Tests: Enhance robustness of stdout driven waiting state in Sikuli based tests. wxPython4
authorEdouard Tisserant <edouard.tisserant@gmail.com>
Sat, 05 Mar 2022 11:14:00 +0100
branchwxPython4
changeset 3437 ce366d67a5b7
parent 3436 ccaabb9da623
child 3438 24fbd4d1fe80
Tests: Enhance robustness of stdout driven waiting state in Sikuli based tests.

Some tests were randomly passing, because from time to time waiting for idle was skiped. It was combination of multiple problems :
- buffering on stdout (now use readline + flush for each write to log)
- it is sometime required to wait for activity before waiting for timeout added "WaitForChangeAndIdle" to "stdoutIdleObserver"
BeremizIDE.py
tests/ide_tests/edit_project.sikuli/edit_project.py
tests/ide_tests/sikuliberemiz.py
--- a/BeremizIDE.py	Mon Feb 28 21:53:14 2022 +0100
+++ b/BeremizIDE.py	Sat Mar 05 11:14:00 2022 +0100
@@ -136,6 +136,7 @@
     def write(self, s, style=None):
         if self.logf is not None:
             self.logf.write(s)
+            self.logf.flush()
         self.StackLock.acquire()
         self.stack.append((s, style))
         self.StackLock.release()
--- a/tests/ide_tests/edit_project.sikuli/edit_project.py	Mon Feb 28 21:53:14 2022 +0100
+++ b/tests/ide_tests/edit_project.sikuli/edit_project.py	Sat Mar 05 11:14:00 2022 +0100
@@ -15,46 +15,55 @@
 proc,app = StartBeremizApp(exemple="python")
 
 # To detect when actions did finish because IDE content isn't changing
-# idle = IDEIdleObserver(app)
-# screencap based idle detection was making many false positive. Test is more stable with stdout based idle detection
+idle = IDEIdleObserver(app)
 
 doubleClick("1646062660770.png")
 
+idle.Wait(1,15)
+
 click("1646066794902.png")
 
+idle.Wait(1,15)
+
 type(Key.DOWN * 10, Key.CTRL)
 
+idle.Wait(1,15)
+
 doubleClick("1646066996620.png")
 
+idle.Wait(1,15)
+
 type(Key.TAB*3)  # select text content
 
-type("'sys.stdout.write(\"EDIT TEST OK\")'")
+type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
 
 type(Key.ENTER)
 
+idle.Wait(1,15)
+
+k = KBDShortcut(app)
+
+k.Save()
+
+del idle
+
 stdoutIdle = stdoutIdleObserver(proc)
 
-# To send keyboard shortuts
-k = KBDShortcut(app)
-
 k.Clean()
 
-stdoutIdle.Wait(2,15)
+stdoutIdle.WaitForChangeAndIdle(2,15)
 
-k.Save()
 k.Build()
 
-stdoutIdle.Wait(2,15)
+stdoutIdle.WaitForChangeAndIdle(2,15)
 
 k.Connect()
 
-stdoutIdle.Wait(2,15)
+stdoutIdle.WaitForChangeAndIdle(2,15)
 
 k.Transfer()
 
-stdoutIdle.Wait(2,15)
-
-#del idle
+stdoutIdle.WaitForChangeAndIdle(2,15)
 
 del stdoutIdle
 
--- a/tests/ide_tests/sikuliberemiz.py	Mon Feb 28 21:53:14 2022 +0100
+++ b/tests/ide_tests/sikuliberemiz.py	Sat Mar 05 11:14:00 2022 +0100
@@ -4,6 +4,7 @@
 import sys
 import subprocess
 from threading import Thread, Event
+from time import time as timesec
 
 typeof=type
 
@@ -142,7 +143,7 @@
             period (int): how many seconds with no change to consider idle
             timeout (int): how long to wait for idle, in seconds
         """
-        c = timeout/period
+        c = max(timeout/period,1)
         while c > 0:
             self.idechanged = False
             wait(period)
@@ -165,15 +166,37 @@
         self.proc = proc
         self.stdoutchanged = False
 
+        self.changes = 0
+        self.last_change_count = 0
+
+        self.event = Event()
+
         self.thread = Thread(target = self._waitStdoutProc).start()
 
     def _waitStdoutProc(self):
         while True:
-            a = self.proc.stdout.read(1)
+            a = self.proc.stdout.readline()
             if len(a) == 0 or a is None: 
                 break
-            sys.stdout.write(a)
-            self.idechanged = True
+            # sys.stdout.write(a)
+            self.changes = self.changes + 1
+            self.event.set()
+
+    def WaitForChangeAndIdle(self, period, timeout):
+        """
+        Wait for IDE'stdout to start changing
+        Parameters: 
+            timeout (int): how long to wait for change, in seconds
+        """
+        start_time = timesec()
+        if self.changes == self.last_change_count:
+            if self.event.wait(timeout):
+                self.event.clear()
+                self.last_change_count = self.changes
+            else:
+                raise Exception("Stdout didn't become active before timeout")
+
+        self.Wait(period, timeout - (timesec() - start_time))
 
     def Wait(self, period, timeout):
         """
@@ -182,11 +205,12 @@
             period (int): how many seconds with no change to consider idle
             timeout (int): how long to wait for idle, in seconds
         """
-        c = timeout/period
+        c = max(timeout/period, 1)
         while c > 0:
-            self.idechanged = False
+            changes = self.changes
             wait(period)
-            if not self.idechanged:
+            if self.changes == changes:
+                self.last_change_count = self.changes
                 break
             c = c - 1
 
@@ -204,8 +228,9 @@
             a = proc.stdout.readline()
             if len(a) == 0 or a is None: 
                 raise Exception("App finished before producing expected stdout pattern")
-            sys.stdout.write(a)
+            # sys.stdout.write(a)
             if a.find(pattern) >= 0:
+                sys.stdout.write("found pattern in '" + a +"'")
                 found = found + 1
                 if found >= count:
                     success_event.set()