util/ProcessLogger.py
author Edouard Tisserant <edouard@beremiz.fr>
Mon, 22 Jul 2024 16:13:27 +0200
changeset 3996 4eb23bb4bc2f
parent 3892 1675b5533e9e
child 3965 6c3469588c12
permissions -rw-r--r--
MQTT: Implements reconnecting in publish thread in case thread is waken-up but client is disconnected.

Note: paho's lostConnection callback got already disabled in previous commit.
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
     1
#!/usr/bin/env python
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
     2
# -*- coding: utf-8 -*-
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
     3
1571
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
     4
# This file is part of Beremiz, a Integrated Development Environment for
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
     5
# programming IEC 61131-3 automates supporting plcopen standard and CanFestival.
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
     6
#
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
     7
# Copyright (C) 2007: Edouard TISSERANT and Laurent BESSARD
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
     8
#
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
     9
# See COPYING file for copyrights details.
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    10
#
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    11
# This program is free software; you can redistribute it and/or
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    12
# modify it under the terms of the GNU General Public License
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    13
# as published by the Free Software Foundation; either version 2
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    14
# of the License, or (at your option) any later version.
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    15
#
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    16
# This program is distributed in the hope that it will be useful,
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    17
# but WITHOUT ANY WARRANTY; without even the implied warranty of
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    18
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    19
# GNU General Public License for more details.
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    20
#
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    21
# You should have received a copy of the GNU General Public License
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    22
# along with this program; if not, write to the Free Software
486f94a8032c fix license notices in source files and license files under GPLv2+
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1539
diff changeset
    23
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
    24
1881
091005ec69c4 fix pylint py3k conversion warning: "(no-absolute-import) import missing `from __future__ import absolute_import`"
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1878
diff changeset
    25
1732
94ffe74e6895 clean-up: fix PEP8 E401 multiple imports on one line
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1730
diff changeset
    26
import os
94ffe74e6895 clean-up: fix PEP8 E401 multiple imports on one line
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1730
diff changeset
    27
import sys
94ffe74e6895 clean-up: fix PEP8 E401 multiple imports on one line
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1730
diff changeset
    28
import subprocess
94ffe74e6895 clean-up: fix PEP8 E401 multiple imports on one line
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1730
diff changeset
    29
import ctypes
2730
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
    30
import time
3892
1675b5533e9e Fix argument lexer in ProcessLogger, replace it with shlex standard python module.
Edouard Tisserant <edouard@beremiz.fr>
parents: 3816
diff changeset
    31
import shlex
2730
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
    32
from threading import Timer, Lock, Thread, Semaphore, Condition
1919
ccea0fa6ea91 Another set of meaningless changes to satisfy PEP8 and PyLint.
Edouard Tisserant
parents: 1883
diff changeset
    33
import signal
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
    34
2671
30493ff3a23a Debug : when BEREMIZ_DEBUG file exist, display all commands issued in console.
Edouard Tisserant
parents: 1919
diff changeset
    35
_debug = os.path.exists("BEREMIZ_DEBUG")
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
    36
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
    37
class outputThread(Thread):
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    38
    """
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    39
    Thread is used to print the output of a command to the stdout
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    40
    """
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    41
    def __init__(self, Proc, fd, callback=None, endcallback=None):
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
    42
        Thread.__init__(self)
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    43
        self.killed = False
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    44
        self.finished = False
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    45
        self.retval = None
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    46
        self.Proc = Proc
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    47
        self.callback = callback
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    48
        self.endcallback = endcallback
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    49
        self.fd = fd
3722
ffce28009ad3 IDE: prevent IDE/CLI process surviving to sys.exit() by using deamon threads to monitor external processes output streams.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 3483
diff changeset
    50
        self.daemon = True
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
    51
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    52
    def run(self):
162
bf3eac08a96b Try to fix strange wxPopen behavior. Feedback appreciated.
etisserant
parents: 154
diff changeset
    53
        outchunk = None
149
fc7fe0de9143 Tried to fix Linux behaviour of wxPopen.py
etisserant
parents: 130
diff changeset
    54
        self.retval = None
1739
ec153828ded2 clean-up: fix PEP8 E203 whitespace before ':' and whitespace before ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1736
diff changeset
    55
        while self.retval is None and not self.killed:
1507
d7f474d10210 fix issue with sometimes wrong return code of ProcessLogger
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1506
diff changeset
    56
            if self.endcallback:
d7f474d10210 fix issue with sometimes wrong return code of ProcessLogger
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1506
diff changeset
    57
                self.retval = self.Proc.poll()
d7f474d10210 fix issue with sometimes wrong return code of ProcessLogger
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1506
diff changeset
    58
            else:
d7f474d10210 fix issue with sometimes wrong return code of ProcessLogger
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1506
diff changeset
    59
                self.retval = self.Proc.returncode
1730
64d8f52bc8c8 clean-up for PEP8: fix W291 trailing whitespace
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1581
diff changeset
    60
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    61
            outchunk = self.fd.readline()
1756
08e4394ff4fb clean-up: fix PEP8 E701 multiple statements on one line (colon)
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1746
diff changeset
    62
            if self.callback:
08e4394ff4fb clean-up: fix PEP8 E701 multiple statements on one line (colon)
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1746
diff changeset
    63
                self.callback(outchunk)
1739
ec153828ded2 clean-up: fix PEP8 E203 whitespace before ':' and whitespace before ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1736
diff changeset
    64
        while outchunk != '' and not self.killed:
162
bf3eac08a96b Try to fix strange wxPopen behavior. Feedback appreciated.
etisserant
parents: 154
diff changeset
    65
            outchunk = self.fd.readline()
1756
08e4394ff4fb clean-up: fix PEP8 E701 multiple statements on one line (colon)
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1746
diff changeset
    66
            if self.callback:
08e4394ff4fb clean-up: fix PEP8 E701 multiple statements on one line (colon)
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1746
diff changeset
    67
                self.callback(outchunk)
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    68
        if self.endcallback:
130
9af34a1d33b7 fixed short process wainting bug. Seems wait() fail when process already finisshed... TO BE CONFIRMED.
greg
parents: 128
diff changeset
    69
            try:
232
83cc1a306560 add call to join method for stdout/stderr threads
greg
parents: 224
diff changeset
    70
                err = self.Proc.wait()
1780
c52d1460cea8 clean-up: fix PEP8 E722 do not use bare except'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1759
diff changeset
    71
            except Exception:
149
fc7fe0de9143 Tried to fix Linux behaviour of wxPopen.py
etisserant
parents: 130
diff changeset
    72
                err = self.retval
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    73
            self.finished = True
421
c9ec111ad275 Bugs with logger and stand-alone PluginRoot fixed
laurent
parents: 361
diff changeset
    74
            self.endcallback(self.Proc.pid, err)
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
    75
1736
7e61baa047f0 clean-up: fix PEP8 E302 expected 2 blank lines, found 1
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1732
diff changeset
    76
1831
56b48961cc68 fix (old-style-class) Old-style class defined error for most parts of
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1780
diff changeset
    77
class ProcessLogger(object):
1744
69dfdb26f600 clean-up: fix PEP8 E251 unexpected spaces around keyword / parameter equals
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1742
diff changeset
    78
    def __init__(self, logger, Command, finish_callback=None,
69dfdb26f600 clean-up: fix PEP8 E251 unexpected spaces around keyword / parameter equals
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1742
diff changeset
    79
                 no_stdout=False, no_stderr=False, no_gui=True,
69dfdb26f600 clean-up: fix PEP8 E251 unexpected spaces around keyword / parameter equals
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1742
diff changeset
    80
                 timeout=None, outlimit=None, errlimit=None,
69dfdb26f600 clean-up: fix PEP8 E251 unexpected spaces around keyword / parameter equals
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1742
diff changeset
    81
                 endlog=None, keyword=None, kill_it=False, cwd=None,
3786
50935528d313 allow passing environment variables to ProcessLogger
GP Orcullo <kinsamanka@gmail.com>
parents: 3755
diff changeset
    82
                 encoding=None, output_encoding=None, env=None):
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    83
        self.logger = logger
424
86a7c1d11bbd support for passing list command to ProcessLogger
greg
parents: 421
diff changeset
    84
        if not isinstance(Command, list):
86a7c1d11bbd support for passing list command to ProcessLogger
greg
parents: 421
diff changeset
    85
            self.Command_str = Command
3892
1675b5533e9e Fix argument lexer in ProcessLogger, replace it with shlex standard python module.
Edouard Tisserant <edouard@beremiz.fr>
parents: 3816
diff changeset
    86
            self.Command = shlex.split(Command)
1675b5533e9e Fix argument lexer in ProcessLogger, replace it with shlex standard python module.
Edouard Tisserant <edouard@beremiz.fr>
parents: 3816
diff changeset
    87
        else:
1675b5533e9e Fix argument lexer in ProcessLogger, replace it with shlex standard python module.
Edouard Tisserant <edouard@beremiz.fr>
parents: 3816
diff changeset
    88
            self.Command = Command
424
86a7c1d11bbd support for passing list command to ProcessLogger
greg
parents: 421
diff changeset
    89
            self.Command_str = subprocess.list2cmdline(self.Command)
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
    90
1415
c411fc7246eb Allow specification of command line arguments encoding in ProcessLogger
Edouard Tisserant
parents: 1407
diff changeset
    91
        fsencoding = sys.getfilesystemencoding()
1883
20ec80d6fd70 add possibility to specify subprocess stdout encoding
Surkov Sergey <surkovsv93@gmail.com>
parents: 1882
diff changeset
    92
        self.output_encoding = output_encoding
1415
c411fc7246eb Allow specification of command line arguments encoding in ProcessLogger
Edouard Tisserant
parents: 1407
diff changeset
    93
c411fc7246eb Allow specification of command line arguments encoding in ProcessLogger
Edouard Tisserant
parents: 1407
diff changeset
    94
        if encoding is None:
c411fc7246eb Allow specification of command line arguments encoding in ProcessLogger
Edouard Tisserant
parents: 1407
diff changeset
    95
            encoding = fsencoding
3750
f62625418bff automated conversion using 2to3-3.9 tool
GP Orcullo <kinsamanka@gmail.com>
parents: 3722
diff changeset
    96
        self.Command = [self.Command[0].encode(fsencoding)]+[x.encode(encoding) for x in self.Command[1:]]
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
    97
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    98
        self.finish_callback = finish_callback
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
    99
        self.no_stdout = no_stdout
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   100
        self.no_stderr = no_stderr
111
e2e498333fbc fixed display/hide console when launch external programs
greg
parents: 110
diff changeset
   101
        self.startupinfo = None
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   102
        self.errlen = 0
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   103
        self.outlen = 0
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   104
        self.errlimit = errlimit
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   105
        self.outlimit = outlimit
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   106
        self.exitcode = None
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   107
        self.outdata = []
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   108
        self.errdata = []
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   109
        self.keyword = keyword
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   110
        self.kill_it = kill_it
1730
64d8f52bc8c8 clean-up for PEP8: fix W291 trailing whitespace
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1581
diff changeset
   111
        self.startsem = Semaphore(0)
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   112
        self.finishsem = Semaphore(0)
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   113
        self.endlock = Lock()
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
   114
1742
92932cd370a4 clean-up: fix PEP8 E225 missing whitespace around operator
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1741
diff changeset
   115
        popenargs = {
1878
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   116
            "cwd":    os.getcwd() if cwd is None else cwd,
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   117
            "stdin":  subprocess.PIPE,
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   118
            "stdout": subprocess.PIPE,
3786
50935528d313 allow passing environment variables to ProcessLogger
GP Orcullo <kinsamanka@gmail.com>
parents: 3755
diff changeset
   119
            "stderr": subprocess.PIPE,
50935528d313 allow passing environment variables to ProcessLogger
GP Orcullo <kinsamanka@gmail.com>
parents: 3755
diff changeset
   120
            "env":    env
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   121
        }
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
   122
1919
ccea0fa6ea91 Another set of meaningless changes to satisfy PEP8 and PyLint.
Edouard Tisserant
parents: 1883
diff changeset
   123
        if no_gui and os.name in ("nt", "ce"):
111
e2e498333fbc fixed display/hide console when launch external programs
greg
parents: 110
diff changeset
   124
            self.startupinfo = subprocess.STARTUPINFO()
e2e498333fbc fixed display/hide console when launch external programs
greg
parents: 110
diff changeset
   125
            self.startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
128
3db703a78e9c fixed subprocess launching on linux (avoid use of undefied self.startupinfo and use use Shell=True (bash will split arguments))
greg
parents: 112
diff changeset
   126
            popenargs["startupinfo"] = self.startupinfo
1919
ccea0fa6ea91 Another set of meaningless changes to satisfy PEP8 and PyLint.
Edouard Tisserant
parents: 1883
diff changeset
   127
        elif os.name == 'posix':
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   128
            popenargs["shell"] = False
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
   129
1506
b9b8978dbc9d Fix error about missing attribute 'timeout' that happens sometimes during compilation
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1476
diff changeset
   130
        if timeout:
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   131
            self.timeout = Timer(timeout, self.endlog)
1506
b9b8978dbc9d Fix error about missing attribute 'timeout' that happens sometimes during compilation
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1476
diff changeset
   132
            self.timeout.start()
b9b8978dbc9d Fix error about missing attribute 'timeout' that happens sometimes during compilation
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1476
diff changeset
   133
        else:
b9b8978dbc9d Fix error about missing attribute 'timeout' that happens sometimes during compilation
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1476
diff changeset
   134
            self.timeout = None
1730
64d8f52bc8c8 clean-up for PEP8: fix W291 trailing whitespace
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1581
diff changeset
   135
3483
e25f1cf69df9 IDE: fix exception happening in debug mode when logging process termination.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 3297
diff changeset
   136
        if _debug and self.logger:
3297
7e59bd180bc6 IDE: systematically log command when launching process (was not showing in case of LOCAL:// runtime being launched)
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2733
diff changeset
   137
            self.logger.write("(DEBUG) launching:\n" + self.Command_str + "\n")
7e59bd180bc6 IDE: systematically log command when launching process (was not showing in case of LOCAL:// runtime being launched)
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2733
diff changeset
   138
3755
ca814b175391 fix string encoding
GP Orcullo <kinsamanka@gmail.com>
parents: 3750
diff changeset
   139
        self.Proc = subprocess.Popen(self.Command, encoding="utf-8", **popenargs)
112
fa0eaeaa9012 Re-enabled stderr
etisserant
parents: 111
diff changeset
   140
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   141
        self.outt = outputThread(
1878
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   142
            self.Proc,
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   143
            self.Proc.stdout,
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   144
            self.output,
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   145
            self.finish)
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   146
        self.outt.start()
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
   147
112
fa0eaeaa9012 Re-enabled stderr
etisserant
parents: 111
diff changeset
   148
        self.errt = outputThread(
1878
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   149
            self.Proc,
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   150
            self.Proc.stderr,
fb73a6b6622d fix pylint warning '(bad-continuation) Wrong hanging indentation before block'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1850
diff changeset
   151
            self.errors)
232
83cc1a306560 add call to join method for stdout/stderr threads
greg
parents: 224
diff changeset
   152
        self.errt.start()
1519
f7ac0d11118e fix race condition in ProcessLogger
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1516
diff changeset
   153
        self.startsem.release()
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
   154
2730
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   155
        self.spinwakeuplock = Lock()
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   156
        self.spinwakeupcond = Condition(self.spinwakeuplock)
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   157
        self.spinwakeuptimer = None
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   158
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   159
    def output(self, v):
1883
20ec80d6fd70 add possibility to specify subprocess stdout encoding
Surkov Sergey <surkovsv93@gmail.com>
parents: 1882
diff changeset
   160
        if v and self.output_encoding:
20ec80d6fd70 add possibility to specify subprocess stdout encoding
Surkov Sergey <surkovsv93@gmail.com>
parents: 1882
diff changeset
   161
            v = v.decode(self.output_encoding)
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   162
        self.outdata.append(v)
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   163
        self.outlen += 1
2712
a00f41d097f3 Removed harmful assert in ProcessLogger.
Edouard Tisserant
parents: 2671
diff changeset
   164
        if self.logger and not self.no_stdout:
686
e4e1da75d411 More robust Logger, now resist to flooding.
Edouard Tisserant
parents: 424
diff changeset
   165
            self.logger.write(v)
1742
92932cd370a4 clean-up: fix PEP8 E225 missing whitespace around operator
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1741
diff changeset
   166
        if (self.keyword and v.find(self.keyword) != -1) or (self.outlimit and self.outlen > self.outlimit):
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   167
            self.endlog()
1407
cf3d2b53dd68 Organized controller's _Build sub-methods layout. Many (unwanted) white space changes.
Edouard Tisserant
parents: 958
diff changeset
   168
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   169
    def errors(self, v):
1883
20ec80d6fd70 add possibility to specify subprocess stdout encoding
Surkov Sergey <surkovsv93@gmail.com>
parents: 1882
diff changeset
   170
        if v and self.output_encoding:
20ec80d6fd70 add possibility to specify subprocess stdout encoding
Surkov Sergey <surkovsv93@gmail.com>
parents: 1882
diff changeset
   171
            v = v.decode(self.output_encoding)
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   172
        self.errdata.append(v)
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   173
        self.errlen += 1
2712
a00f41d097f3 Removed harmful assert in ProcessLogger.
Edouard Tisserant
parents: 2671
diff changeset
   174
        if self.logger and not self.no_stderr:
686
e4e1da75d411 More robust Logger, now resist to flooding.
Edouard Tisserant
parents: 424
diff changeset
   175
            self.logger.write_warning(v)
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   176
        if self.errlimit and self.errlen > self.errlimit:
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   177
            self.endlog()
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
   178
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   179
    def log_the_end(self, ecode, pid):
2671
30493ff3a23a Debug : when BEREMIZ_DEBUG file exist, display all commands issued in console.
Edouard Tisserant
parents: 1919
diff changeset
   180
        if self.logger is not None:
30493ff3a23a Debug : when BEREMIZ_DEBUG file exist, display all commands issued in console.
Edouard Tisserant
parents: 1919
diff changeset
   181
            self.logger.write(self.Command_str + "\n")
30493ff3a23a Debug : when BEREMIZ_DEBUG file exist, display all commands issued in console.
Edouard Tisserant
parents: 1919
diff changeset
   182
            self.logger.write_warning(_("exited with status {a1} (pid {a2})\n").format(a1=str(ecode), a2=str(pid)))
232
83cc1a306560 add call to join method for stdout/stderr threads
greg
parents: 224
diff changeset
   183
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   184
    def finish(self, pid, ecode):
1730
64d8f52bc8c8 clean-up for PEP8: fix W291 trailing whitespace
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1581
diff changeset
   185
        # avoid running function before start is finished
1519
f7ac0d11118e fix race condition in ProcessLogger
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1516
diff changeset
   186
        self.startsem.acquire()
1882
3707240447f8 fix deadlock if method kill() is called after process is already finished
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1881
diff changeset
   187
        self.startsem.release()
1476
49f1763a5613 fixes sometimes happened error during project compilation
Andrey Skvortsov <andrej.skvortzov@gmail.com
parents: 1415
diff changeset
   188
        if self.timeout:
49f1763a5613 fixes sometimes happened error during project compilation
Andrey Skvortsov <andrej.skvortzov@gmail.com
parents: 1415
diff changeset
   189
            self.timeout.cancel()
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   190
        self.exitcode = ecode
3297
7e59bd180bc6 IDE: systematically log command when launching process (was not showing in case of LOCAL:// runtime being launched)
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2733
diff changeset
   191
        if self.exitcode != 0:
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   192
            self.log_the_end(ecode, pid)
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   193
        if self.finish_callback is not None:
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   194
            self.finish_callback(self, ecode, pid)
1516
9db0b436fbb3 fix bug: Beremiz IDE hangs during launching Beremiz_service
andrey@new-nest
parents: 1507
diff changeset
   195
        self.errt.join()
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   196
        self.finishsem.release()
2730
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   197
        self.spinwakeup()
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
   198
1740
b789b695b5c6 clean-up: fix PEP8 E231 missing whitespace after ':' or ','
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1739
diff changeset
   199
    def kill(self, gently=True):
1539
8e5c03798abc fix syncronization issue during killing ProcessLoggers on IDE close
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1519
diff changeset
   200
        # avoid running kill before start is finished
8e5c03798abc fix syncronization issue during killing ProcessLoggers on IDE close
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1519
diff changeset
   201
        self.startsem.acquire()
8e5c03798abc fix syncronization issue during killing ProcessLoggers on IDE close
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1519
diff changeset
   202
        self.startsem.release()
1730
64d8f52bc8c8 clean-up for PEP8: fix W291 trailing whitespace
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1581
diff changeset
   203
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   204
        self.outt.killed = True
112
fa0eaeaa9012 Re-enabled stderr
etisserant
parents: 111
diff changeset
   205
        self.errt.killed = True
1919
ccea0fa6ea91 Another set of meaningless changes to satisfy PEP8 and PyLint.
Edouard Tisserant
parents: 1883
diff changeset
   206
        if os.name in ("nt", "ce"):
110
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   207
            PROCESS_TERMINATE = 1
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   208
            handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, False, self.Proc.pid)
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   209
            ctypes.windll.kernel32.TerminateProcess(handle, -1)
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   210
            ctypes.windll.kernel32.CloseHandle(handle)
a05e8b30c024 Fixed way apps are launched in parralel with single log window... Tested in win32 only.
etisserant
parents: 89
diff changeset
   211
        else:
235
a66e150f2888 Improved debug data feedback.
etisserant
parents: 232
diff changeset
   212
            if gently:
1919
ccea0fa6ea91 Another set of meaningless changes to satisfy PEP8 and PyLint.
Edouard Tisserant
parents: 1883
diff changeset
   213
                sig = signal.SIGTERM
235
a66e150f2888 Improved debug data feedback.
etisserant
parents: 232
diff changeset
   214
            else:
1919
ccea0fa6ea91 Another set of meaningless changes to satisfy PEP8 and PyLint.
Edouard Tisserant
parents: 1883
diff changeset
   215
                sig = signal.SIGKILL
154
f3134b2c6d92 Fixed killing app on Linux in wxPopen. Do not use shell anymore. Command line is splitted into args, taking care of double and simple cotes. To be tested on win32.
etisserant
parents: 151
diff changeset
   216
            try:
235
a66e150f2888 Improved debug data feedback.
etisserant
parents: 232
diff changeset
   217
                os.kill(self.Proc.pid, sig)
1780
c52d1460cea8 clean-up: fix PEP8 E722 do not use bare except'
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1759
diff changeset
   218
            except Exception:
154
f3134b2c6d92 Fixed killing app on Linux in wxPopen. Do not use shell anymore. Command line is splitted into args, taking care of double and simple cotes. To be tested on win32.
etisserant
parents: 151
diff changeset
   219
                pass
232
83cc1a306560 add call to join method for stdout/stderr threads
greg
parents: 224
diff changeset
   220
        self.outt.join()
83cc1a306560 add call to join method for stdout/stderr threads
greg
parents: 224
diff changeset
   221
        self.errt.join()
79
ae06c2da83f7 Some window related enhancements
etisserant
parents: 70
diff changeset
   222
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   223
    def endlog(self):
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   224
        if self.endlock.acquire(False):
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   225
            if not self.outt.finished and self.kill_it:
1757
0de89da92ee0 clean-up: fix PEP8 E111 indentation is not a multiple of four
Andrey Skvortsov <andrej.skvortzov@gmail.com>
parents: 1756
diff changeset
   226
                self.kill()
1516
9db0b436fbb3 fix bug: Beremiz IDE hangs during launching Beremiz_service
andrey@new-nest
parents: 1507
diff changeset
   227
            self.finishsem.release()
2730
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   228
            self.spinwakeup()
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   229
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   230
    def spinwakeup(self):
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   231
        with self.spinwakeuplock:
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   232
            if self.spinwakeuptimer is not None:
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   233
                self.spinwakeuptimer.cancel()
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   234
            self.spinwakeuptimer = None
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   235
            self.spinwakeupcond.notify()
89
0ab2868c6aa6 Added right aligment of parameteres blocks
etisserant
parents: 79
diff changeset
   236
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   237
    def spin(self):
2730
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   238
        start = time.time()
2733
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   239
        if self.logger:
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   240
            while not self.finishsem.acquire(0):
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   241
                with self.spinwakeuplock:
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   242
                    self.spinwakeuptimer = Timer(0.1, self.spinwakeup)
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   243
                    self.spinwakeuptimer.start()
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   244
                    self.spinwakeupcond.wait()
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   245
                    self.logger.progress("%.3fs"%(time.time() - start))
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   246
        else:
244598a6a0d6 Allow using spin() from ProcessLogger even with no logger given.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2730
diff changeset
   247
            self.finishsem.acquire()
2730
ce21ce181fdb IDE: Process Logging : Add annotation in log so that user can see build is still alive and how long external process takes.
Edouard Tisserant <edouard.tisserant@gmail.com>
parents: 2712
diff changeset
   248
704
5993b16fe2d0 More stable ProcessLogger.spin()
Edouard Tisserant
parents: 686
diff changeset
   249
        return [self.exitcode, "".join(self.outdata), "".join(self.errdata)]