edouard@3424: #! gmake
edouard@3424: 
edouard@3424: # beremiz/tests/Makefile :
edouard@3424: #
edouard@3424: #   Makefile to prepare and run Beremiz tests.
edouard@3424: #
edouard@3424: #   For developper to:
edouard@3424: #       - quickly run a test (TDD) on current code 
edouard@3424: #       - write new tests, debug existing tests 
edouard@3424: #
edouard@3424: #     Use cases :
edouard@3424: #
edouard@3424: #       run given tests
edouard@3424: #           $ make run_python_exemple.sikuli
edouard@3424: #
edouard@3424: #       run tests from particular test classes
edouard@3424: #           $ make ide_tests
edouard@3424: #
edouard@3424: #       run one particular test in a Xnest window
edouard@3424: #           $ make xnest_run_python_exemple.sikuli
edouard@3424: #
edouard@3424: #       run Xnest window with just xterm
edouard@3424: #           $ make xnest_xterm
edouard@3424: #
edouard@3424: #       run Xnest window with sikuli IDE and xterm
edouard@3430: #           $ make xnest_sikuli
edouard@3424: #
edouard@3424: #       build minimal beremiz and matiec to run tests
edouard@3549: #           $ make built_apps
edouard@3424: #
edouard@3424: #   For CI/CD scripts to catch and report all failures. Use cases :
edouard@3424: #
edouard@3424: #       run all tests
edouard@3424: #           $ make
edouard@3424: #
edouard@3424: #   
edouard@3424: #   Test results, and other test byproducts are in $(test_dir), 
edouard@3424: #   $(test_dir) defaults to $(HOME)/test and can be overloaded:
edouard@3424: #       $ make test_dir=${HOME}/other_test_dir
edouard@3424: #
edouard@3428: #   Makefile attemps to use xvfb-run to run each test individually with its own
edouard@3428: #   X server instance. This behavior can be overloaded
edouard@3428: #       $ DISPLAY=:42 make xserver_command='echo "Using $DISPLAY X Server !";'
edouard@3424: #
edouard@3430: #   Matiec and Beremiz code are expected to be clean, ready to build
edouard@3424: #   Any change in Matiec directory triggers rebuild of matiec.
edouard@3424: #   Any change in Matiec and Beremiz directory triggers copy of source code
edouard@3424: #   to $(test_dir)/build.
edouard@3424: #
edouard@3438: #   BEREMIZPYTHONPATH is expected to be absolute path to python interpreter
edouard@3424: #
edouard@3424: #   Please note:
edouard@3424: #       In order to run asside a freshly build Matiec, tested beremiz instance
edouard@3424: #       needs to run on code from $(test_dir)/build/beremiz, a fresh copy
edouard@3424: #       of the Beremiz directory $(src)/beremiz, where we run tests from.
edouard@3424: #   
edouard@3424: 
edouard@3424: all: source_check cli_tests ide_tests runtime_tests 
edouard@3424: 
edouard@3430: # Variable $(src) is directory such that executed 
edouard@3430: # $(src)/Makefile is this file.
edouard@3424: src := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
edouard@3430: 
edouard@3430: # $(workspace) is directory containing this project
edouard@3424: workspace ?= $(abspath $(src)/../..)
edouard@3424: 
edouard@3424: test_dir ?= $(HOME)/test
edouard@3424: build_dir = $(test_dir)/build
edouard@3424: 
edouard@3541: #
edouard@3541: # SOURCE and BUILD
edouard@3541: #
edouard@3541: 
edouard@3549: BUILT_PROJECTS=beremiz matiec open62541
edouard@3424: 
edouard@3432: tar_opts=--absolute-names --exclude=.hg --exclude=.git --exclude=.*.pyc --exclude=.*.swp
edouard@3432: 
edouard@3424: # sha1 checksum of source is used to force copy/compile on each change
edouard@3424: 
edouard@3424: define make_checksum_assign
edouard@3432: $(1)_checksum = $(shell tar $(tar_opts) -c $(workspace)/$(1) | sha1sum | cut -d ' ' -f 1)
edouard@3424: endef
edouard@3549: $(foreach project,$(BUILT_PROJECTS),$(eval $(call make_checksum_assign,$(project))))
edouard@3424: 
edouard@3424: $(build_dir):
edouard@3424: 	mkdir -p $(build_dir)
edouard@3424: 
edouard@3424: define make_src_rule
edouard@3550: $(build_dir)/$(1)/$($(1)_checksum).sha1: | $(build_dir) $(workspace)/$(1)
edouard@3424: 	rm -rf $(build_dir)/$(1)
edouard@3432: 	tar -C $(workspace) $(tar_opts) -c $(1) | tar -C $(build_dir) -x
edouard@3424: 	touch $$@
edouard@3424: endef
edouard@3549: $(foreach project,$(BUILT_PROJECTS),$(eval $(call make_src_rule,$(project))))
edouard@3424: 
edouard@3550: $(build_dir)/matiec/iec2c: $(build_dir)/matiec/$(matiec_checksum).sha1
edouard@3424: 	cd $(build_dir)/matiec && \
edouard@3424:     autoreconf -i && \
edouard@3424:     ./configure && \
edouard@3424:     make
edouard@3424: 
edouard@3550: $(build_dir)/open62541/build/bin/libopen62541.a: $(build_dir)/open62541/$(open62541_checksum).sha1
edouard@3549: 	cd $(build_dir)/open62541 && \
edouard@3549:     rm -rf build && mkdir build && cd build && \
edouard@3552: 	cmake .. && \
edouard@3549: 	make
edouard@3549: 
edouard@3549: built_apps: $(build_dir)/matiec/iec2c $(build_dir)/beremiz/$(beremiz_checksum).sha1 $(build_dir)/open62541/build/bin/libopen62541.a
edouard@3424: 	touch $@
edouard@3424: 
edouard@3541: define log_command
edouard@3541: 	$(call $(1),$(2)) | tee test_stdout.txt; exit $$$${PIPESTATUS[0]}
edouard@3541: endef
edouard@3541: 
edouard@3541: define prep_test
edouard@3541: 	rm -rf $(test_dir)/$(1)_results
edouard@3541: 	mkdir $(test_dir)/$(1)_results
edouard@3541: 	cd $(test_dir)/$(1)_results
edouard@3541: endef
edouard@3541: 
edouard@3541: #
edouard@3541: # IDE TESTS
edouard@3541: #
edouard@3541: 
edouard@3438: ide_test_dir = $(src)/ide_tests
edouard@3438: sikuli_ide_tests = $(subst $(ide_test_dir)/,,$(wildcard $(ide_test_dir)/*.sikuli))
edouard@3438: pytest_ide_tests = $(subst $(ide_test_dir)/,,$(wildcard $(ide_test_dir)/*.pytest))
edouard@3424: 
edouard@3438: define sikuli_idetest_command
edouard@3541: 	(fluxbox >/dev/null 2>&1 &); BEREMIZPATH=$(build_dir)/beremiz sikulix -r $(src)/ide_tests/$(1)
edouard@3424: endef
edouard@3424: 
edouard@3438: 
edouard@3438: DELAY=400
edouard@3438: KILL_DELAY=430
edouard@3438: PYTEST=$(dir $(BEREMIZPYTHONPATH))/pytest
edouard@3438: define pytest_idetest_command
edouard@3541: 	(fluxbox >/dev/null 2>&1 &); PYTHONPATH=$(ide_test_dir) timeout -k $(KILL_DELAY) $(DELAY) $(PYTEST) --maxfail=1 --timeout=100  $(src)/ide_tests/$(1)
edouard@3438: endef
edouard@3438: 
edouard@3424: # Xnest based interactive sessions for tests edit and debug. 
edouard@3424: # Would be nice with something equivalent to xvfb-run, waiting for USR1.
edouard@3424: # Arbitrary "sleep 1" is probably enough for interactive use
edouard@3424: define xnest_run
edouard@3424: 	Xnest :42 -geometry 1920x1080+0+0 & export xnestpid=$$!; sleep 1; DISPLAY=:42 $(1); export res=$$?; kill $${xnestpid} 2>/dev/null; exit $${res}
edouard@3424: endef
edouard@3424: 
edouard@3428: xserver_command ?= xvfb-run -s '-screen 0 1920x1080x24'
edouard@3428: 
edouard@3424: define make_idetest_rule
edouard@3549: $(test_dir)/$(1)_results/.passed: built_apps
edouard@3541: 	$(call prep_test,$(1)); $(xserver_command) bash -c '$(call log_command,$(2),$(1))'
edouard@3424: 	touch $$@
edouard@3424: 
edouard@3428: # Manually invoked rule {testname}.sikuli
edouard@3541: $(1): $(test_dir)/$(1)_results/.passed
edouard@3424: 
edouard@3424: # Manually invoked rule xnest_{testname}.sikuli
edouard@3424: # runs test in xnest so that one can see what happens
edouard@3549: xnest_$(1): built_apps
edouard@3541: 	$(call prep_test,$(1)); $$(call xnest_run, bash -c '$(call log_command,$(2),$(1))')
edouard@3541: 
edouard@3541: ide_tests_targets += $(test_dir)/$(1)_results/.passed
edouard@3424: endef
edouard@3438: $(foreach idetest,$(sikuli_ide_tests),$(eval $(call make_idetest_rule,$(idetest),sikuli_idetest_command)))
edouard@3438: $(foreach idetest,$(pytest_ide_tests),$(eval $(call make_idetest_rule,$(idetest),pytest_idetest_command)))
edouard@3424: 
edouard@3424: ide_tests : $(ide_tests_targets)
edouard@3435: 	echo "$(ide_tests_targets) : Passed"
edouard@3424: 
edouard@3549: xnest_xterm: built_apps
edouard@3424: 	$(call xnest_run, bash -c '(fluxbox &);xterm')
edouard@3424: 
edouard@3549: xnest_sikuli: built_apps
edouard@3432: 	$(call xnest_run, bash -c '(fluxbox &);(BEREMIZPATH=$(build_dir)/beremiz xterm -e sikulix &);xterm')
edouard@3424: 
edouard@3549: xvfb_sikuli: built_apps
edouard@3435: 	echo "******************************************"
edouard@3435: 	echo "On host, run 'xvncviewer 127.0.0.1:5900' to see sikuli X session"
edouard@3435: 	echo "Docker container must be created with TESTDEBUG=YES. For example :"
edouard@3527: 	echo "./clean_docker_container.sh && ./build_docker_image.sh && TESTDEBUG=YES ./create_docker_container.sh && ./do_test_in_docker.sh xvfb_sikuli"
edouard@3435: 	echo "******************************************"
edouard@3435: 	$(xserver_command) bash -c '(fluxbox &);(x11vnc &);(BEREMIZPATH=$(build_dir)/beremiz xterm -e sikulix &);xterm'
edouard@3424: 
edouard@3541: #
edouard@3541: # CLI TESTS
edouard@3541: #
edouard@3541: 
edouard@3541: cli_test_dir = $(src)/cli_tests
edouard@3541: cli_tests = $(subst $(cli_test_dir)/,,$(wildcard $(cli_test_dir)/*.bash))
edouard@3541: 
edouard@3541: define clitest_command
edouard@3541: 	BEREMIZPATH=$(build_dir)/beremiz source $(src)/cli_tests/$(1)
edouard@3541: endef
edouard@3541: 
edouard@3541: define make_clitest_rule
edouard@3549: $(test_dir)/$(1)_results/.passed: built_apps
edouard@3541: 	$(call prep_test,$(1)); bash -c '$(call log_command,$(2),$(1))'
edouard@3541: 	touch $$@
edouard@3541: 
edouard@3541: # Manually invoked rule
edouard@3541: $(1): $(test_dir)/$(1)_results/.passed
edouard@3541: 
edouard@3541: cli_tests_targets += $(test_dir)/$(1)_results/.passed
edouard@3541: endef
edouard@3541: $(foreach clitest,$(cli_tests),$(eval $(call make_clitest_rule,$(clitest),clitest_command)))
edouard@3541: 
edouard@3541: cli_tests: $(cli_tests_targets)
edouard@3541: 	echo "$(cli_tests_targets) : Passed"
edouard@3541: 
edouard@3541: clean_results:
edouard@3541: 	rm -rf $(test_dir)/*_results
edouard@3541: 
edouard@3541: clean: clean_results
edouard@3541: 	rm -rf $(build_dir)
edouard@3424: 
edouard@3424: 
edouard@3424: # TODOs 
edouard@3424: 
edouard@3424: source_check:
edouard@3424: 	echo TODO $@
edouard@3424: 
edouard@3424: runtime_tests:
edouard@3424: 	echo TODO $@
edouard@3424: 
edouard@3424: 
edouard@3424: