# HG changeset patch
# User Edouard Tisserant <edouard.tisserant@gmail.com>
# Date 1649310032 -7200
# Node ID 65c5f66e9298f1252a6b45d41e024554441703b0
# Parent  de8cc85b688abc30ab86da9045325b524ac8bf38
Tests: add HTML report generation and a workaround to bad exception handling in sikuli.

In case of exception in python code, and since a thread is running
to observe stdout, sikuli was never terminated after an exception.
Unfortunately sys.exepthook doesn't work in that version of jython/sikuli.
Test are now written inside functions witch are passed to run_test to deal
with exception.

diff -r de8cc85b688a -r 65c5f66e9298 tests/ide_tests/edit_project.sikuli/edit_project.py
--- a/tests/ide_tests/edit_project.sikuli/edit_project.py	Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/edit_project.sikuli/edit_project.py	Thu Apr 07 07:40:32 2022 +0200
@@ -9,62 +9,55 @@
 addImportPath(os.path.dirname(getBundlePath()))
 
 # common test definitions module
-from sikuliberemiz import *
+from sikuliberemiz import run_test
 
-# Start the app
-app = BeremizApp(exemple="python")
+def test(app):
 
-app.doubleClick("1646062660770.png")
+    app.doubleClick("1646062660770.png")
 
-app.WaitIdleUI()
+    app.WaitIdleUI()
 
-app.click("example")
+    app.click("example")
 
-app.WaitIdleUI()
+    app.WaitIdleUI()
 
-app.type(Key.DOWN * 10, Key.CTRL)
+    app.type(Key.DOWN * 10, Key.CTRL)
 
-app.WaitIdleUI()
+    app.WaitIdleUI()
 
-app.doubleClick("1646066996620.png")
+    app.doubleClick("1646066996620.png")
 
-app.WaitIdleUI()
+    app.WaitIdleUI()
 
-app.type(Key.TAB*3)  # select text content
+    app.type(Key.TAB*3)  # select text content
 
-app.type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
+    app.type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
 
-app.type(Key.ENTER)
+    app.type(Key.ENTER)
 
-app.WaitIdleUI()
+    app.WaitIdleUI()
 
-app.k.Save()
+    app.k.Save()
 
-app.k.Clean()
+    app.k.Clean()
 
-app.waitForChangeAndIdleStdout()
+    app.waitForChangeAndIdleStdout()
 
-app.k.Build()
+    app.k.Build()
 
-app.waitForChangeAndIdleStdout()
+    app.waitForChangeAndIdleStdout()
 
-app.k.Connect()
+    app.k.Connect()
 
-app.waitForChangeAndIdleStdout()
+    app.waitForChangeAndIdleStdout()
 
-app.k.Transfer()
+    app.k.Transfer()
 
-app.waitForChangeAndIdleStdout()
+    app.waitForChangeAndIdleStdout()
 
-app.k.Run()
+    app.k.Run()
 
-# wait 10 seconds for 10 patterns
-found = app.waitPatternInStdout("EDIT TEST OK", 10)
+    # wait 10 seconds for 10 patterns
+    return app.waitPatternInStdout("EDIT TEST OK", 10)
 
-app.close()
-
-if found:
-    exit(0)
-else:
-    exit(1)
-
+run_test(test, exemple="python")
diff -r de8cc85b688a -r 65c5f66e9298 tests/ide_tests/new_project.sikuli/new_project.py
--- a/tests/ide_tests/new_project.sikuli/new_project.py	Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/new_project.sikuli/new_project.py	Thu Apr 07 07:40:32 2022 +0200
@@ -11,128 +11,122 @@
 # common test definitions module
 from sikuliberemiz import *
 
-# Start the app without any project given
-app = BeremizApp()
+def test(app):
+    
+    new_project_path = os.path.join(os.path.abspath(os.path.curdir), "new_test_project")
+    
+    # New project path must exist (usually created in directory selection dialog)
+    os.mkdir(new_project_path)
+    
+    app.WaitIdleUI()
+    
+    # Create new project (opens new project directory selection dialog)
+    app.k.New()
+    
+    app.WaitIdleUI()
+    
+    # Move to "Home" section of file selecor, otherwise address is 
+    # "file ignored" at first run
+    app.type("f", Key.CTRL)
+    app.type(Key.ESC)
+    app.type(Key.TAB)
+    
+    # Enter directory by name
+    app.k.Address()
+    
+    # Fill address bar
+    app.type(new_project_path + Key.ENTER)
+    
+    app.WaitIdleUI()
+    
+    # When prompted for creating first program select type ST
+    app.type(Key.TAB*4)  # go to lang dropdown
+    app.type(Key.DOWN*2) # change selected language
+    app.type(Key.ENTER)  # validate
+    
+    app.WaitIdleUI()
+    
+    # Name created program
+    app.type("Test program")
+    
+    app.WaitIdleUI()
+    
+    # Focus on Variable grid
+    app.type(Key.TAB*4)
+    
+    # Add 2 variables
+    app.type(Key.ADD*2)
+    
+    # Focus on ST text
+    app.WaitIdleUI()
+    
+    app.type(Key.TAB*8)
+    
+    app.type("""\
+    LocalVar0 := LocalVar1;
+    {printf("Test OK\\n");fflush(stdout);}
+    """)
+    
+    app.k.Save()
+    
+    # Close ST POU
+    app.type("w", Key.CTRL)
+    
+    app.WaitIdleUI()
+    
+    # Focus project tree and select root item
+    app.type(Key.TAB)
+    
+    app.type(Key.LEFT)
+    
+    app.type(Key.UP)
+    
+    # Edit root item
+    app.type(Key.ENTER)
+    
+    app.WaitIdleUI()
+    
+    # Switch to config tab
+    app.type(Key.RIGHT*2)
+    
+    # Focus on URI
+    app.type(Key.TAB)
+    
+    # Set URI
+    app.type("LOCAL://")
+    
+    # FIXME: Select other field to ensure URI is validated
+    app.type(Key.TAB)
+    
+    app.k.Save()
+    
+    # Close project config editor
+    app.type("w", Key.CTRL)
+    
+    app.WaitIdleUI()
+    
+    # Focus seems undefined at that time (FIXME)
+    # Force focussing on "something" so that next shortcut is taken
+    app.type(Key.TAB)
+    
+    app.waitIdleStdout()
+    
+    app.k.Build()
+    
+    app.waitIdleStdout(5,30)
+    
+    app.k.Connect()
+    
+    app.waitIdleStdout()
+    
+    app.k.Transfer()
+    
+    app.waitIdleStdout()
+    
+    app.k.Run()
+    
+    # wait 10 seconds
+    return app.waitPatternInStdout("Test OK", 10)
 
-new_project_path = os.path.join(os.path.abspath(os.path.curdir), "new_test_project")
 
-# New project path must exist (usually created in directory selection dialog)
-os.mkdir(new_project_path)
-
-app.WaitIdleUI()
-
-# Create new project (opens new project directory selection dialog)
-app.k.New()
-
-app.WaitIdleUI()
-
-# Move to "Home" section of file selecor, otherwise address is 
-# "file ignored" at first run
-app.type("f", Key.CTRL)
-app.type(Key.ESC)
-app.type(Key.TAB)
-
-# Enter directory by name
-app.k.Address()
-
-# Fill address bar
-app.type(new_project_path + Key.ENTER)
-
-app.WaitIdleUI()
-
-# When prompted for creating first program select type ST
-app.type(Key.TAB*4)  # go to lang dropdown
-app.type(Key.DOWN*2) # change selected language
-app.type(Key.ENTER)  # validate
-
-app.WaitIdleUI()
-
-# Name created program
-app.type("Test program")
-
-app.WaitIdleUI()
-
-# Focus on Variable grid
-app.type(Key.TAB*4)
-
-# Add 2 variables
-app.type(Key.ADD*2)
-
-# Focus on ST text
-app.WaitIdleUI()
-
-app.type(Key.TAB*8)
-
-app.type("""\
-LocalVar0 := LocalVar1;
-{printf("Test OK\\n");fflush(stdout);}
-""")
-
-app.k.Save()
-
-# Close ST POU
-app.type("w", Key.CTRL)
-
-app.WaitIdleUI()
-
-# Focus project tree and select root item
-app.type(Key.TAB)
-
-app.type(Key.LEFT)
-
-app.type(Key.UP)
-
-# Edit root item
-app.type(Key.ENTER)
-
-app.WaitIdleUI()
-
-# Switch to config tab
-app.type(Key.RIGHT*2)
-
-# Focus on URI
-app.type(Key.TAB)
-
-# Set URI
-app.type("LOCAL://")
-
-# FIXME: Select other field to ensure URI is validated
-app.type(Key.TAB)
-
-app.k.Save()
-
-# Close project config editor
-app.type("w", Key.CTRL)
-
-app.WaitIdleUI()
-
-# Focus seems undefined at that time (FIXME)
-# Force focussing on "something" so that next shortcut is taken
-app.type(Key.TAB)
-
-app.waitIdleStdout()
-
-app.k.Build()
-
-app.waitIdleStdout(5,30)
-
-app.k.Connect()
-
-app.waitIdleStdout()
-
-app.k.Transfer()
-
-app.waitIdleStdout()
-
-app.k.Run()
-
-# wait 10 seconds
-found = app.waitPatternInStdout("Test OK", 10)
-
-app.close()
-
-if found:
-    exit(0)
-else:
-    exit(1)
-
+run_test(test)
diff -r de8cc85b688a -r 65c5f66e9298 tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py
--- a/tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py	Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py	Thu Apr 07 07:40:32 2022 +0200
@@ -11,34 +11,29 @@
 # common test definitions module
 from sikuliberemiz import *
 
-# Start the app
-app = BeremizApp(exemple="python")
+def test(app):
+    # Start the app
+    
+    app.k.Clean()
+    
+    app.waitForChangeAndIdleStdout()
+    
+    app.k.Build()
+    
+    app.waitForChangeAndIdleStdout()
+    
+    app.k.Connect()
+    
+    app.waitForChangeAndIdleStdout()
+    
+    app.k.Transfer()
+    
+    app.waitForChangeAndIdleStdout()
+    
+    app.k.Run()
+      
+    # wait 10 seconds for 10 Grumpfs
+    return app.waitPatternInStdout("Grumpf", 10, 10)
+    
+run_test(test, exemple="python")
 
-app.k.Clean()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Build()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Connect()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Transfer()
-
-app.waitForChangeAndIdleStdout()
-
-app.k.Run()
-
-# wait 10 seconds for 10 Grumpfs
-found = app.waitPatternInStdout("Grumpf", 10, 10)
-
-app.close()
-
-if found:
-    exit(0)
-else:
-    exit(1)
-
diff -r de8cc85b688a -r 65c5f66e9298 tests/ide_tests/sikuliberemiz.py
--- a/tests/ide_tests/sikuliberemiz.py	Tue Mar 29 08:50:01 2022 +0200
+++ b/tests/ide_tests/sikuliberemiz.py	Thu Apr 07 07:40:32 2022 +0200
@@ -3,6 +3,7 @@
 import os
 import sys
 import subprocess
+import traceback
 from threading import Thread, Event, Lock
 from time import time as timesec
 
@@ -33,17 +34,17 @@
              "Address":  ("l",sikuli.Key.CTRL)}  # to reach address bar in GTK's file selector
 
     def __init__(self, app):
-        self.app = app.sikuliapp
+        self.app = app
     
     def __getattr__(self, name):
         fkey = self.fkeys[name]
         if type(fkey) != tuple:
             fkey = (fkey,)
-        app = self.app
 
         def PressShortCut():
-            app.focus()
+            self.app.sikuliapp.focus()
             sikuli.type(*fkey)
+            self.app.ReportText("Sending " + name + " shortcut")
 
         return PressShortCut
 
@@ -86,6 +87,8 @@
                 break
             c = c - 1
 
+        self.ReportScreenShot("UI is idle" if c != 0 else "UI is not idle")
+
         if c == 0:
             raise Exception("Window did not idle before timeout")
 
@@ -113,6 +116,7 @@
             if len(a) == 0 or a is None: 
                 break
             sys.stdout.write(a)
+            self.ReportOutput(a)
             self.event.set()
             if self.pattern is not None and a.find(self.pattern) >= 0:
                 sys.stdout.write("found pattern in '" + a +"'")
@@ -126,7 +130,11 @@
         """
         start_time = timesec()
 
-        if self.event.wait(timeout):
+        wait_result = self.event.wait(timeout)
+
+        self.ReportScreenShot("stdout changed" if wait_result else "stdout didn't change")
+
+        if wait_result:
             self.event.clear()
         else:
             raise Exception("Stdout didn't become active before timeout")
@@ -148,8 +156,11 @@
                 self.event.clear()
             else:
                 # timeout -> no event -> idle -> exit
+                self.ReportScreenShot("stdout is idle")
                 return True
 
+        self.ReportScreenShot("stdout did not idle")
+
         raise Exception("Stdout did not idle before timeout")
 
     def waitPatternInStdout(self, pattern, timeout, count=1):
@@ -170,6 +181,7 @@
                 if found >= count:
                     break
         self.pattern = None
+        self.ReportScreenShot("found pattern" if res else "pattern not found")
         return res
 
 class BeremizApp(IDEIdleObserver, stdoutIdleObserver):
@@ -185,6 +197,21 @@
                 Sikuli App class instance
         """
 
+        self.screenshotnum = 0
+        self.starttime = timesec()
+        self.screen = sikuli.Screen()
+
+        self.report = open("report.html", "w")
+        self.report.write("""<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta name="color-scheme" content="light dark">
+    <title>Test report</title>
+  </head>
+  <body>
+""")
+
         command = [python_bin, opj(beremiz_path,"Beremiz.py"), "--log=/dev/stdout"]
 
         if exemple is not None:
@@ -198,6 +225,8 @@
         #              - use wmctrl to find IDE window details and maximize it
         #              - pass exact window title to App class constructor
 
+        self.ReportText("Launching " + repr(command))
+
         self.proc = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=0)
 
         # Window are macthed against process' PID
@@ -236,12 +265,21 @@
         stdoutIdleObserver.__init__(self)
 
         # stubs for common sikuli calls to allow adding hooks later
-        for n in ["click","doubleClick","type"]:
-            setattr(self, n, getattr(sikuli, n))
+        for name in ["click","doubleClick","type"]:
+            def makeMyMeth(n):
+                def myMeth(*args, **kwargs):
+                    getattr(sikuli, n)(*args, **kwargs)
+                    self.ReportScreenShot(n + "(" + repr(args) + "," + repr(kwargs) + ")")
+                return myMeth
+            setattr(self, name, makeMyMeth(name))
 
     def close(self):
         self.sikuliapp.close()
         self.sikuliapp = None
+        self.report.write("""
+  </body>
+</html>""")
+        self.report.close()
 
     def __del__(self):
         if self.sikuliapp is not None:
@@ -249,3 +287,44 @@
         IDEIdleObserver.__del__(self)
         stdoutIdleObserver.__del__(self)
 
+    def ReportScreenShot(self, msg):
+        elapsed = "%.3fs: "%(timesec() - self.starttime)
+        fname = "capture"+str(self.screenshotnum)+".png"
+        cap = self.screen.capture(self.r)
+        cap.save(".", fname)
+        # self.report.write("ReportScreenShot " + msg + " " + fname + "\n")
+        self.screenshotnum = self.screenshotnum + 1
+        self.report.write( "<p>" + elapsed + msg + "<img src=\""+ fname + "\">" + "</p>")
+
+    def ReportText(self, text):
+        elapsed = "%.3fs: "%(timesec() - self.starttime)
+        self.report.write("<p>" + elapsed + text + "</p>")
+
+    def ReportOutput(self, text):
+        elapsed = "%.3fs: "%(timesec() - self.starttime)
+        self.report.write("<pre>" + elapsed + text + "</pre>")
+
+
+def run_test(func, *args, **kwargs):
+    app = BeremizApp(*args, **kwargs)
+    try:
+        success = func(app)
+    except:
+        # sadly, sys.excepthook is broken in sikuli/jython 
+        # purpose of this run_test function is to work around it.
+        # and catch exception cleanly anyhow
+        e_type, e_value, e_traceback = sys.exc_info()
+        err_msg = "\n".join(traceback.format_exception(e_type, e_value, e_traceback))
+        sys.stdout.write(err_msg)
+        app.ReportOutput(err_msg)
+        success = False
+
+    app.close()
+
+    if success:
+        sikuli.exit(0)
+    else:
+        sikuli.exit(1)
+
+
+