Tests: Enhance robustness of stdout driven waiting state in Sikuli based tests.
Some tests were randomly passing, because from time to time waiting for idle was skiped. It was combination of multiple problems :
- buffering on stdout (now use readline + flush for each write to log)
- it is sometime required to wait for activity before waiting for timeout added "WaitForChangeAndIdle" to "stdoutIdleObserver"
--- a/BeremizIDE.py Mon Feb 28 21:53:14 2022 +0100
+++ b/BeremizIDE.py Sat Mar 05 11:14:00 2022 +0100
@@ -136,6 +136,7 @@
def write(self, s, style=None):
if self.logf is not None:
self.logf.write(s)
+ self.logf.flush()
self.StackLock.acquire()
self.stack.append((s, style))
self.StackLock.release()
--- a/tests/ide_tests/edit_project.sikuli/edit_project.py Mon Feb 28 21:53:14 2022 +0100
+++ b/tests/ide_tests/edit_project.sikuli/edit_project.py Sat Mar 05 11:14:00 2022 +0100
@@ -15,46 +15,55 @@
proc,app = StartBeremizApp(exemple="python")
# To detect when actions did finish because IDE content isn't changing
-# idle = IDEIdleObserver(app)
-# screencap based idle detection was making many false positive. Test is more stable with stdout based idle detection
+idle = IDEIdleObserver(app)
doubleClick("1646062660770.png")
+idle.Wait(1,15)
+
click("1646066794902.png")
+idle.Wait(1,15)
+
type(Key.DOWN * 10, Key.CTRL)
+idle.Wait(1,15)
+
doubleClick("1646066996620.png")
+idle.Wait(1,15)
+
type(Key.TAB*3) # select text content
-type("'sys.stdout.write(\"EDIT TEST OK\")'")
+type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
type(Key.ENTER)
+idle.Wait(1,15)
+
+k = KBDShortcut(app)
+
+k.Save()
+
+del idle
+
stdoutIdle = stdoutIdleObserver(proc)
-# To send keyboard shortuts
-k = KBDShortcut(app)
-
k.Clean()
-stdoutIdle.Wait(2,15)
+stdoutIdle.WaitForChangeAndIdle(2,15)
-k.Save()
k.Build()
-stdoutIdle.Wait(2,15)
+stdoutIdle.WaitForChangeAndIdle(2,15)
k.Connect()
-stdoutIdle.Wait(2,15)
+stdoutIdle.WaitForChangeAndIdle(2,15)
k.Transfer()
-stdoutIdle.Wait(2,15)
-
-#del idle
+stdoutIdle.WaitForChangeAndIdle(2,15)
del stdoutIdle
--- a/tests/ide_tests/sikuliberemiz.py Mon Feb 28 21:53:14 2022 +0100
+++ b/tests/ide_tests/sikuliberemiz.py Sat Mar 05 11:14:00 2022 +0100
@@ -4,6 +4,7 @@
import sys
import subprocess
from threading import Thread, Event
+from time import time as timesec
typeof=type
@@ -142,7 +143,7 @@
period (int): how many seconds with no change to consider idle
timeout (int): how long to wait for idle, in seconds
"""
- c = timeout/period
+ c = max(timeout/period,1)
while c > 0:
self.idechanged = False
wait(period)
@@ -165,15 +166,37 @@
self.proc = proc
self.stdoutchanged = False
+ self.changes = 0
+ self.last_change_count = 0
+
+ self.event = Event()
+
self.thread = Thread(target = self._waitStdoutProc).start()
def _waitStdoutProc(self):
while True:
- a = self.proc.stdout.read(1)
+ a = self.proc.stdout.readline()
if len(a) == 0 or a is None:
break
- sys.stdout.write(a)
- self.idechanged = True
+ # sys.stdout.write(a)
+ self.changes = self.changes + 1
+ self.event.set()
+
+ def WaitForChangeAndIdle(self, period, timeout):
+ """
+ Wait for IDE'stdout to start changing
+ Parameters:
+ timeout (int): how long to wait for change, in seconds
+ """
+ start_time = timesec()
+ if self.changes == self.last_change_count:
+ if self.event.wait(timeout):
+ self.event.clear()
+ self.last_change_count = self.changes
+ else:
+ raise Exception("Stdout didn't become active before timeout")
+
+ self.Wait(period, timeout - (timesec() - start_time))
def Wait(self, period, timeout):
"""
@@ -182,11 +205,12 @@
period (int): how many seconds with no change to consider idle
timeout (int): how long to wait for idle, in seconds
"""
- c = timeout/period
+ c = max(timeout/period, 1)
while c > 0:
- self.idechanged = False
+ changes = self.changes
wait(period)
- if not self.idechanged:
+ if self.changes == changes:
+ self.last_change_count = self.changes
break
c = c - 1
@@ -204,8 +228,9 @@
a = proc.stdout.readline()
if len(a) == 0 or a is None:
raise Exception("App finished before producing expected stdout pattern")
- sys.stdout.write(a)
+ # sys.stdout.write(a)
if a.find(pattern) >= 0:
+ sys.stdout.write("found pattern in '" + a +"'")
found = found + 1
if found >= count:
success_event.set()