Tests: add HTML report generation and a workaround to bad exception handling in sikuli.
In case of exception in python code, and since a thread is running
to observe stdout, sikuli was never terminated after an exception.
Unfortunately sys.exepthook doesn't work in that version of jython/sikuli.
Test are now written inside functions witch are passed to run_test to deal
with exception.
--- a/tests/ide_tests/edit_project.sikuli/edit_project.py Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/edit_project.sikuli/edit_project.py Thu Apr 07 07:40:32 2022 +0200
@@ -9,62 +9,55 @@
addImportPath(os.path.dirname(getBundlePath()))
# common test definitions module
-from sikuliberemiz import *
+from sikuliberemiz import run_test
-# Start the app
-app = BeremizApp(exemple="python")
+def test(app):
-app.doubleClick("1646062660770.png")
+ app.doubleClick("1646062660770.png")
-app.WaitIdleUI()
+ app.WaitIdleUI()
-app.click("example")
+ app.click("example")
-app.WaitIdleUI()
+ app.WaitIdleUI()
-app.type(Key.DOWN * 10, Key.CTRL)
+ app.type(Key.DOWN * 10, Key.CTRL)
-app.WaitIdleUI()
+ app.WaitIdleUI()
-app.doubleClick("1646066996620.png")
+ app.doubleClick("1646066996620.png")
-app.WaitIdleUI()
+ app.WaitIdleUI()
-app.type(Key.TAB*3) # select text content
+ app.type(Key.TAB*3) # select text content
-app.type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
+ app.type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
-app.type(Key.ENTER)
+ app.type(Key.ENTER)
-app.WaitIdleUI()
+ app.WaitIdleUI()
-app.k.Save()
+ app.k.Save()
-app.k.Clean()
+ app.k.Clean()
-app.waitForChangeAndIdleStdout()
+ app.waitForChangeAndIdleStdout()
-app.k.Build()
+ app.k.Build()
-app.waitForChangeAndIdleStdout()
+ app.waitForChangeAndIdleStdout()
-app.k.Connect()
+ app.k.Connect()
-app.waitForChangeAndIdleStdout()
+ app.waitForChangeAndIdleStdout()
-app.k.Transfer()
+ app.k.Transfer()
-app.waitForChangeAndIdleStdout()
+ app.waitForChangeAndIdleStdout()
-app.k.Run()
+ app.k.Run()
-# wait 10 seconds for 10 patterns
-found = app.waitPatternInStdout("EDIT TEST OK", 10)
+ # wait 10 seconds for 10 patterns
+ return app.waitPatternInStdout("EDIT TEST OK", 10)
-app.close()
-
-if found:
- exit(0)
-else:
- exit(1)
-
+run_test(test, exemple="python")
--- a/tests/ide_tests/new_project.sikuli/new_project.py Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/new_project.sikuli/new_project.py Thu Apr 07 07:40:32 2022 +0200
@@ -11,128 +11,122 @@
# common test definitions module
from sikuliberemiz import *
-# Start the app without any project given
-app = BeremizApp()
+def test(app):
+
+ new_project_path = os.path.join(os.path.abspath(os.path.curdir), "new_test_project")
+
+ # New project path must exist (usually created in directory selection dialog)
+ os.mkdir(new_project_path)
+
+ app.WaitIdleUI()
+
+ # Create new project (opens new project directory selection dialog)
+ app.k.New()
+
+ app.WaitIdleUI()
+
+ # Move to "Home" section of file selecor, otherwise address is
+ # "file ignored" at first run
+ app.type("f", Key.CTRL)
+ app.type(Key.ESC)
+ app.type(Key.TAB)
+
+ # Enter directory by name
+ app.k.Address()
+
+ # Fill address bar
+ app.type(new_project_path + Key.ENTER)
+
+ app.WaitIdleUI()
+
+ # When prompted for creating first program select type ST
+ app.type(Key.TAB*4) # go to lang dropdown
+ app.type(Key.DOWN*2) # change selected language
+ app.type(Key.ENTER) # validate
+
+ app.WaitIdleUI()
+
+ # Name created program
+ app.type("Test program")
+
+ app.WaitIdleUI()
+
+ # Focus on Variable grid
+ app.type(Key.TAB*4)
+
+ # Add 2 variables
+ app.type(Key.ADD*2)
+
+ # Focus on ST text
+ app.WaitIdleUI()
+
+ app.type(Key.TAB*8)
+
+ app.type("""\
+ LocalVar0 := LocalVar1;
+ {printf("Test OK\\n");fflush(stdout);}
+ """)
+
+ app.k.Save()
+
+ # Close ST POU
+ app.type("w", Key.CTRL)
+
+ app.WaitIdleUI()
+
+ # Focus project tree and select root item
+ app.type(Key.TAB)
+
+ app.type(Key.LEFT)
+
+ app.type(Key.UP)
+
+ # Edit root item
+ app.type(Key.ENTER)
+
+ app.WaitIdleUI()
+
+ # Switch to config tab
+ app.type(Key.RIGHT*2)
+
+ # Focus on URI
+ app.type(Key.TAB)
+
+ # Set URI
+ app.type("LOCAL://")
+
+ # FIXME: Select other field to ensure URI is validated
+ app.type(Key.TAB)
+
+ app.k.Save()
+
+ # Close project config editor
+ app.type("w", Key.CTRL)
+
+ app.WaitIdleUI()
+
+ # Focus seems undefined at that time (FIXME)
+ # Force focussing on "something" so that next shortcut is taken
+ app.type(Key.TAB)
+
+ app.waitIdleStdout()
+
+ app.k.Build()
+
+ app.waitIdleStdout(5,30)
+
+ app.k.Connect()
+
+ app.waitIdleStdout()
+
+ app.k.Transfer()
+
+ app.waitIdleStdout()
+
+ app.k.Run()
+
+ # wait 10 seconds
+ return app.waitPatternInStdout("Test OK", 10)
-new_project_path = os.path.join(os.path.abspath(os.path.curdir), "new_test_project")
-# New project path must exist (usually created in directory selection dialog)
-os.mkdir(new_project_path)
-
-app.WaitIdleUI()
-
-# Create new project (opens new project directory selection dialog)
-app.k.New()
-
-app.WaitIdleUI()
-
-# Move to "Home" section of file selecor, otherwise address is
-# "file ignored" at first run
-app.type("f", Key.CTRL)
-app.type(Key.ESC)
-app.type(Key.TAB)
-
-# Enter directory by name
-app.k.Address()
-
-# Fill address bar
-app.type(new_project_path + Key.ENTER)
-
-app.WaitIdleUI()
-
-# When prompted for creating first program select type ST
-app.type(Key.TAB*4) # go to lang dropdown
-app.type(Key.DOWN*2) # change selected language
-app.type(Key.ENTER) # validate
-
-app.WaitIdleUI()
-
-# Name created program
-app.type("Test program")
-
-app.WaitIdleUI()
-
-# Focus on Variable grid
-app.type(Key.TAB*4)
-
-# Add 2 variables
-app.type(Key.ADD*2)
-
-# Focus on ST text
-app.WaitIdleUI()
-
-app.type(Key.TAB*8)
-
-app.type("""\
-LocalVar0 := LocalVar1;
-{printf("Test OK\\n");fflush(stdout);}
-""")
-
-app.k.Save()
-
-# Close ST POU
-app.type("w", Key.CTRL)
-
-app.WaitIdleUI()
-
-# Focus project tree and select root item
-app.type(Key.TAB)
-
-app.type(Key.LEFT)
-
-app.type(Key.UP)
-
-# Edit root item
-app.type(Key.ENTER)
-
-app.WaitIdleUI()
-
-# Switch to config tab
-app.type(Key.RIGHT*2)
-
-# Focus on URI
-app.type(Key.TAB)
-
-# Set URI
-app.type("LOCAL://")
-
-# FIXME: Select other field to ensure URI is validated
-app.type(Key.TAB)
-
-app.k.Save()
-
-# Close project config editor
-app.type("w", Key.CTRL)
-
-app.WaitIdleUI()
-
-# Focus seems undefined at that time (FIXME)
-# Force focussing on "something" so that next shortcut is taken
-app.type(Key.TAB)
-
-app.waitIdleStdout()
-
-app.k.Build()
-
-app.waitIdleStdout(5,30)
-
-app.k.Connect()
-
-app.waitIdleStdout()
-
-app.k.Transfer()
-
-app.waitIdleStdout()
-
-app.k.Run()
-
-# wait 10 seconds
-found = app.waitPatternInStdout("Test OK", 10)
-
-app.close()
-
-if found:
- exit(0)
-else:
- exit(1)
-
+run_test(test)
--- a/tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py Thu Apr 07 07:40:32 2022 +0200
@@ -11,34 +11,29 @@
# common test definitions module
from sikuliberemiz import *
-# Start the app
-app = BeremizApp(exemple="python")
+def test(app):
+ # Start the app
+
+ app.k.Clean()
+
+ app.waitForChangeAndIdleStdout()
+
+ app.k.Build()
+
+ app.waitForChangeAndIdleStdout()
+
+ app.k.Connect()
+
+ app.waitForChangeAndIdleStdout()
+
+ app.k.Transfer()
+
+ app.waitForChangeAndIdleStdout()
+
+ app.k.Run()
+
+ # wait 10 seconds for 10 Grumpfs
+ return app.waitPatternInStdout("Grumpf", 10, 10)
+
+run_test(test, exemple="python")
-app.k.Clean()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Build()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Connect()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Transfer()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Run()
-
-# wait 10 seconds for 10 Grumpfs
-found = app.waitPatternInStdout("Grumpf", 10, 10)
-
-app.close()
-
-if found:
- exit(0)
-else:
- exit(1)
-
--- a/tests/ide_tests/sikuliberemiz.py Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/sikuliberemiz.py Thu Apr 07 07:40:32 2022 +0200
@@ -3,6 +3,7 @@
import os
import sys
import subprocess
+import traceback
from threading import Thread, Event, Lock
from time import time as timesec
@@ -33,17 +34,17 @@
"Address": ("l",sikuli.Key.CTRL)} # to reach address bar in GTK's file selector
def __init__(self, app):
- self.app = app.sikuliapp
+ self.app = app
def __getattr__(self, name):
fkey = self.fkeys[name]
if type(fkey) != tuple:
fkey = (fkey,)
- app = self.app
def PressShortCut():
- app.focus()
+ self.app.sikuliapp.focus()
sikuli.type(*fkey)
+ self.app.ReportText("Sending " + name + " shortcut")
return PressShortCut
@@ -86,6 +87,8 @@
break
c = c - 1
+ self.ReportScreenShot("UI is idle" if c != 0 else "UI is not idle")
+
if c == 0:
raise Exception("Window did not idle before timeout")
@@ -113,6 +116,7 @@
if len(a) == 0 or a is None:
break
sys.stdout.write(a)
+ self.ReportOutput(a)
self.event.set()
if self.pattern is not None and a.find(self.pattern) >= 0:
sys.stdout.write("found pattern in '" + a +"'")
@@ -126,7 +130,11 @@
"""
start_time = timesec()
- if self.event.wait(timeout):
+ wait_result = self.event.wait(timeout)
+
+ self.ReportScreenShot("stdout changed" if wait_result else "stdout didn't change")
+
+ if wait_result:
self.event.clear()
else:
raise Exception("Stdout didn't become active before timeout")
@@ -148,8 +156,11 @@
self.event.clear()
else:
# timeout -> no event -> idle -> exit
+ self.ReportScreenShot("stdout is idle")
return True
+ self.ReportScreenShot("stdout did not idle")
+
raise Exception("Stdout did not idle before timeout")
def waitPatternInStdout(self, pattern, timeout, count=1):
@@ -170,6 +181,7 @@
if found >= count:
break
self.pattern = None
+ self.ReportScreenShot("found pattern" if res else "pattern not found")
return res
class BeremizApp(IDEIdleObserver, stdoutIdleObserver):
@@ -185,6 +197,21 @@
Sikuli App class instance
"""
+ self.screenshotnum = 0
+ self.starttime = timesec()
+ self.screen = sikuli.Screen()
+
+ self.report = open("report.html", "w")
+ self.report.write("""<!doctype html>
+<html>
+ <head>
+ <meta charset="utf-8">
+ <meta name="color-scheme" content="light dark">
+ <title>Test report</title>
+ </head>
+ <body>
+""")
+
command = [python_bin, opj(beremiz_path,"Beremiz.py"), "--log=/dev/stdout"]
if exemple is not None:
@@ -198,6 +225,8 @@
# - use wmctrl to find IDE window details and maximize it
# - pass exact window title to App class constructor
+ self.ReportText("Launching " + repr(command))
+
self.proc = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=0)
# Window are macthed against process' PID
@@ -236,12 +265,21 @@
stdoutIdleObserver.__init__(self)
# stubs for common sikuli calls to allow adding hooks later
- for n in ["click","doubleClick","type"]:
- setattr(self, n, getattr(sikuli, n))
+ for name in ["click","doubleClick","type"]:
+ def makeMyMeth(n):
+ def myMeth(*args, **kwargs):
+ getattr(sikuli, n)(*args, **kwargs)
+ self.ReportScreenShot(n + "(" + repr(args) + "," + repr(kwargs) + ")")
+ return myMeth
+ setattr(self, name, makeMyMeth(name))
def close(self):
self.sikuliapp.close()
self.sikuliapp = None
+ self.report.write("""
+ </body>
+</html>""")
+ self.report.close()
def __del__(self):
if self.sikuliapp is not None:
@@ -249,3 +287,44 @@
IDEIdleObserver.__del__(self)
stdoutIdleObserver.__del__(self)
+ def ReportScreenShot(self, msg):
+ elapsed = "%.3fs: "%(timesec() - self.starttime)
+ fname = "capture"+str(self.screenshotnum)+".png"
+ cap = self.screen.capture(self.r)
+ cap.save(".", fname)
+ # self.report.write("ReportScreenShot " + msg + " " + fname + "\n")
+ self.screenshotnum = self.screenshotnum + 1
+ self.report.write( "<p>" + elapsed + msg + "<img src=\""+ fname + "\">" + "</p>")
+
+ def ReportText(self, text):
+ elapsed = "%.3fs: "%(timesec() - self.starttime)
+ self.report.write("<p>" + elapsed + text + "</p>")
+
+ def ReportOutput(self, text):
+ elapsed = "%.3fs: "%(timesec() - self.starttime)
+ self.report.write("<pre>" + elapsed + text + "</pre>")
+
+
+def run_test(func, *args, **kwargs):
+ app = BeremizApp(*args, **kwargs)
+ try:
+ success = func(app)
+ except:
+ # sadly, sys.excepthook is broken in sikuli/jython
+ # purpose of this run_test function is to work around it.
+ # and catch exception cleanly anyhow
+ e_type, e_value, e_traceback = sys.exc_info()
+ err_msg = "\n".join(traceback.format_exception(e_type, e_value, e_traceback))
+ sys.stdout.write(err_msg)
+ app.ReportOutput(err_msg)
+ success = False
+
+ app.close()
+
+ if success:
+ sikuli.exit(0)
+ else:
+ sikuli.exit(1)
+
+
+