Chunk based transfer for PLC binary and extra files, and some collateral code refactoring.
authorEdouard Tisserant
Tue, 04 Dec 2018 11:31:58 +0100
changeset 2463 8742337a9fe3
parent 2462 ed6b0e905fcb
child 2464 10437c6c294e
Chunk based transfer for PLC binary and extra files, and some collateral code refactoring.
ProjectController.py
connectors/ConnectorBase.py
connectors/PYRO/__init__.py
connectors/WAMP/__init__.py
connectors/__init__.py
runtime/PLCObject.py
runtime/Worker.py
targets/toolchain_gcc.py
targets/toolchain_makefile.py
--- a/ProjectController.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/ProjectController.py	Tue Dec 04 11:31:58 2018 +0100
@@ -891,19 +891,6 @@
             self._builder = targetclass(self)
         return self._builder
 
-    def ResetBuildMD5(self):
-        builder = self.GetBuilder()
-        if builder is not None:
-            builder.ResetBinaryCodeMD5()
-        self.EnableMethod("_Transfer", False)
-
-    def GetLastBuildMD5(self):
-        builder = self.GetBuilder()
-        if builder is not None:
-            return builder.GetBinaryCodeMD5()
-        else:
-            return None
-
     #
     #
     #                C CODE GENERATION METHODS
@@ -1131,7 +1118,6 @@
         # If IEC code gen fail, bail out.
         if not IECGenRes:
             self.logger.write_error(_("PLC code generation failed !\n"))
-            self.ResetBuildMD5()
             return False
 
         # Reset variable and program list that are parsed from
@@ -1147,7 +1133,6 @@
         builder = self.GetBuilder()
         if builder is None:
             self.logger.write_error(_("Fatal : cannot get builder.\n"))
-            self.ResetBuildMD5()
             return False
 
         # Build
@@ -1156,9 +1141,9 @@
                 self.logger.write_error(_("C Build failed.\n"))
                 return False
         except Exception:
+            builder.ResetBinaryMD5()
             self.logger.write_error(_("C Build crashed !\n"))
             self.logger.write_error(traceback.format_exc())
-            self.ResetBuildMD5()
             return False
 
         self.logger.write(_("Successfully built.\n"))
@@ -1178,7 +1163,6 @@
             self.logger.write_error(
                 _("Runtime IO extensions C code generation failed !\n"))
             self.logger.write_error(traceback.format_exc())
-            self.ResetBuildMD5()
             return False
 
         # Generate C code and compilation params from liraries
@@ -1189,7 +1173,6 @@
             self.logger.write_error(
                 _("Runtime library extensions C code generation failed !\n"))
             self.logger.write_error(traceback.format_exc())
-            self.ResetBuildMD5()
             return False
 
         self.LocationCFilesAndCFLAGS = LibCFilesAndCFLAGS + \
@@ -1239,7 +1222,6 @@
             except Exception:
                 self.logger.write_error(name + _(" generation failed !\n"))
                 self.logger.write_error(traceback.format_exc())
-                self.ResetBuildMD5()
                 return False
         self.logger.write(_("C code generated successfully.\n"))
         return True
@@ -1820,26 +1802,18 @@
     def CompareLocalAndRemotePLC(self):
         if self._connector is None:
             return
-        # We are now connected. Update button status
-        MD5 = self.GetLastBuildMD5()
+        builder = self.GetBuilder()
+        if builder is None :
+            return
+        MD5 = builder.GetBinaryMD5()
+        if MD5 is None:
+            return
         # Check remote target PLC correspondance to that md5
-        if MD5 is not None:
-            if not self._connector.MatchMD5(MD5):
-                # self.logger.write_warning(
-                # _("Latest build does not match with target, please
-                # transfer.\n"))
-                self.EnableMethod("_Transfer", True)
-            else:
-                # self.logger.write(
-                #     _("Latest build matches target, no transfer needed.\n"))
-                self.EnableMethod("_Transfer", True)
-                # warns controller that program match
-                self.ProgramTransferred()
-                # self.EnableMethod("_Transfer", False)
+        if self._connector.MatchMD5(MD5):
+            self.ProgramTransferred()
         else:
-            # self.logger.write_warning(
-            #     _("Cannot compare latest build to target. Please build.\n"))
-            self.EnableMethod("_Transfer", False)
+            self.logger.write(
+                _("Latest build does not match with connected target.\n"))
 
     def _Disconnect(self):
         self._SetConnector(None)
@@ -1855,8 +1829,13 @@
             else:
                 return
 
-        # Get the last build PLC's
-        MD5 = self.GetLastBuildMD5()
+        builder = self.GetBuilder()
+        if builder is None:
+            self.logger.write_error(_("Fatal : cannot get builder.\n"))
+            return False
+
+        # recover md5 from last build 
+        MD5 = builder.GetBinaryMD5()
 
         # Check if md5 file is empty : ask user to build PLC
         if MD5 is None:
@@ -1869,35 +1848,37 @@
             self.logger.write(
                 _("Latest build already matches current target. Transfering anyway...\n"))
 
-        # Get temprary directory path
+        # purge any non-finished transfer
+        # note: this would abord any runing transfer with error
+        self._connector.PurgeBlobs()
+
+        # transfer extra files
         extrafiles = []
         for extrafilespath in [self._getExtraFilesPath(),
                                self._getProjectFilesPath()]:
 
-            extrafiles.extend(
-                [(name, open(os.path.join(extrafilespath, name),
-                             'rb').read())
-                 for name in os.listdir(extrafilespath)])
+            for name in os.listdir(extrafilespath):
+                extrafiles.append((
+                    name, 
+                    self._connector.BlobFromFile(
+                        os.path.join(extrafilespath, name))))
 
         # Send PLC on target
-        builder = self.GetBuilder()
-        if builder is not None:
-            data = builder.GetBinaryCode()
-            if data is not None:
-                if self._connector.NewPLC(MD5, data, extrafiles) and self.GetIECProgramsAndVariables():
-                    self.UnsubscribeAllDebugIECVariable()
-                    self.ProgramTransferred()
-                    if self.AppFrame is not None:
-                        self.AppFrame.CloseObsoleteDebugTabs()
-                        self.AppFrame.RefreshPouInstanceVariablesPanel()
-                    self.logger.write(_("Transfer completed successfully.\n"))
-                    self.AppFrame.LogViewer.ResetLogCounters()
-                else:
-                    self.logger.write_error(_("Transfer failed\n"))
-                self.HidePLCProgress()
-            else:
-                self.logger.write_error(
-                    _("No PLC to transfer (did build succeed ?)\n"))
+        object_path = builder.GetBinaryPath()
+        object_blob = self._connector.BlobFromFile(object_path)
+
+        if self._connector.NewPLC(MD5, object_blob, extrafiles):
+            self.ProgramTransferred()
+            self.AppFrame.CloseObsoleteDebugTabs()
+            self.AppFrame.LogViewer.ResetLogCounters()
+            if self.GetIECProgramsAndVariables():
+                self.UnsubscribeAllDebugIECVariable()
+                self.AppFrame.RefreshPouInstanceVariablesPanel()
+            self.logger.write(_("Transfer completed successfully.\n"))
+        else:
+            self.logger.write_error(_("Transfer failed\n"))
+
+        self.HidePLCProgress()
 
         wx.CallAfter(self.UpdateMethodsFromPLCStatus)
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/connectors/ConnectorBase.py	Tue Dec 04 11:31:58 2018 +0100
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# See COPYING file for copyrights details.
+
+import md5
+
+class ConnectorBase(object):
+
+    chuncksize = 16384
+    def BlobFromFile(self, filepath): 
+        s = md5.new()
+        blobID = s.digest()  # empty md5, to support empty blob
+        with open(filepath, "rb") as f:
+            while True:
+                chunk = f.read(self.chuncksize) 
+                if len(chunk) == 0: return blobID
+                blobID = self.AppendChunkToBlob(chunk, blobID)
+                s.update(chunk)
+                if blobID != s.digest(): return None
+
--- a/connectors/PYRO/__init__.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/connectors/PYRO/__init__.py	Tue Dec 04 11:31:58 2018 +0100
@@ -150,4 +150,4 @@
                 self.__dict__[attrName] = member
             return member
 
-    return PyroProxyProxy()
+    return PyroProxyProxy
--- a/connectors/WAMP/__init__.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/connectors/WAMP/__init__.py	Tue Dec 04 11:31:58 2018 +0100
@@ -156,10 +156,4 @@
     # TODO : GetPLCID()
     # TODO : PSK.UpdateID()
 
-    # Try to get the proxy object
-    try:
-        return WampPLCObjectProxy()
-    except Exception:
-        confnodesroot.logger.write_error(_("WAMP connection to '%s' failed.\n") % location)
-        confnodesroot.logger.write_error(traceback.format_exc())
-        return None
+    return WampPLCObjectProxy
--- a/connectors/__init__.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/connectors/__init__.py	Tue Dec 04 11:31:58 2018 +0100
@@ -29,6 +29,8 @@
 from __future__ import absolute_import
 from os import listdir, path
 import util.paths as paths
+from connectors.ConnectorBase import ConnectorBase
+from types import ClassType
 
 connectors_packages = ["PYRO","WAMP"]
 
@@ -64,8 +66,8 @@
     Return a connector corresponding to the URI
     or None if cannot connect to URI
     """
-    scheme = uri.split("://")[0].upper()
-    if scheme == "LOCAL":
+    _scheme = uri.split("://")[0].upper()
+    if _scheme == "LOCAL":
         # Local is special case
         # pyro connection to local runtime
         # started on demand, listening on random port
@@ -73,17 +75,21 @@
         runtime_port = confnodesroot.AppFrame.StartLocalRuntime(
             taskbaricon=True)
         uri = "PYROLOC://127.0.0.1:" + str(runtime_port)
-    elif scheme in connectors:
-        pass
-    elif scheme[-1] == 'S' and scheme[:-1] in connectors:
-        scheme = scheme[:-1]
+    elif _scheme in connectors:
+        scheme = _scheme
+    elif _scheme[-1] == 'S' and _scheme[:-1] in connectors:
+        scheme = _scheme[:-1]
     else:
         return None
 
-    # import module according to uri type
-    connectorclass = connectors[scheme]()
-    return connectorclass(uri, confnodesroot)
+    # import module according to uri type and get connector specific baseclass
+    # first call to import the module, 
+    # then call with parameters to create the class
+    connector_specific_class = connectors[scheme]()(uri, confnodesroot)
 
+    # new class inheriting from generic and specific connector base classes
+    return ClassType(_scheme + "_connector", 
+                     (ConnectorBase, connector_specific_class), {})()
 
 def EditorClassFromScheme(scheme):
     _Import_Dialogs()
--- a/runtime/PLCObject.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/runtime/PLCObject.py	Tue Dec 04 11:31:58 2018 +0100
@@ -34,6 +34,9 @@
 import Pyro.core as pyro
 import six
 from six.moves import _thread, xrange
+import md5
+from tempfile import mkstemp
+import shutil
 
 from runtime.typemapping import TypeTranslator
 from runtime.loglevels import LogLevelsDefault, LogLevelsCount
@@ -41,6 +44,8 @@
 from runtime import PlcStatus
 from runtime import MainWorker
 
+empty_md5_digest = md5.new().digest()
+
 if os.name in ("nt", "ce"):
     dlopen = _ctypes.LoadLibrary
     dlclose = _ctypes.FreeLibrary
@@ -74,7 +79,11 @@
 
 class PLCObject(object):
     def __init__(self, WorkingDir, argv, statuschange, evaluator, pyruntimevars):
-        self.workingdir = WorkingDir
+        self.workingdir = WorkingDir # must exits already
+        self.tmpdir = os.path.join(WorkingDir, 'tmp')
+        if os.path.exists(self.tmpdir):
+            shutil.rmtree(self.tmpdir)
+        os.mkdir(self.tmpdir)
         # FIXME : is argv of any use nowadays ?
         self.argv = [WorkingDir] + argv  # force argv[0] to be "path" to exec...
         self.statuschange = statuschange
@@ -91,6 +100,8 @@
         self.TraceLock = Lock()
         self.Traces = []
 
+        self._init_blobs()
+
     # First task of worker -> no @RunInMain
     def AutoLoad(self, autostart):
         # Get the last transfered PLC
@@ -447,8 +458,52 @@
     def GetPLCID(self):
         return getPSKID()
 
-    @RunInMain
-    def NewPLC(self, md5sum, data, extrafiles):
+    def _init_blobs(self):
+        self.blobs = {}
+        if os.path.exists(self.tmpdir):
+            shutil.rmtree(self.tmpdir)
+        os.mkdir(self.tmpdir)
+    
+    @RunInMain
+    def AppendChunkToBlob(self, data, blobID):
+        blob = ((mkstemp(dir=self.tmpdir) if data else None)\
+                   + (md5.new(),)) \
+               if blobID == empty_md5_digest \
+               else self.blobs.pop(blobID, None)
+
+        if blob is None:
+            return None
+
+        fobj, path, md5sum = blob
+        md5sum.update(data)
+        newBlobID = md5sum.digest()
+        if data:
+            os.write(fobj,data)
+            self.blobs[newBlobID] = blob
+        return newBlobID
+
+    @RunInMain
+    def PurgeBlobs(self):
+        for fobj, path, md5sum in self.blobs:
+            os.close(fobj) 
+        self._init_blobs()
+
+    def _BlobAsFile(self, blobID, newpath):
+        blob = self.blobs.pop(blobID, None)
+
+        if blob is None:
+            if blobID == md5.new().digest():
+                # create empty file
+                open(newpath,'r').close()
+                return
+            raise Exception(_("Missing data to create file: {}").format(newpath))
+
+        fobj, path, md5sum = blob
+        os.close(fobj)
+        shutil.move(path, newpath)
+            
+    @RunInMain
+    def NewPLC(self, md5sum, plc_object, extrafiles):
         if self.PLCStatus in [PlcStatus.Stopped, PlcStatus.Empty, PlcStatus.Broken]:
             NewFileName = md5sum + lib_ext
             extra_files_log = os.path.join(self.workingdir, "extra_files.txt")
@@ -458,18 +513,13 @@
                 else None
             new_PLC_filename = os.path.join(self.workingdir, NewFileName)
 
-            # Some platform (i.e. Xenomai) don't like reloading same .so file
-            replace_PLC_shared_object = new_PLC_filename != old_PLC_filename
-
-            if replace_PLC_shared_object:
-                self.UnLoadPLC()
+            self.UnLoadPLC()
 
             self.LogMessage("NewPLC (%s)" % md5sum)
             self.PLCStatus = PlcStatus.Empty
 
             try:
-                if replace_PLC_shared_object:
-                    os.remove(old_PLC_filename)
+                os.remove(old_PLC_filename)
                 for filename in open(extra_files_log, "rt").readlines() + [extra_files_log]:
                     try:
                         os.remove(os.path.join(self.workingdir, filename.strip()))
@@ -480,17 +530,16 @@
 
             try:
                 # Create new PLC file
-                if replace_PLC_shared_object:
-                    open(new_PLC_filename, 'wb').write(data)
+                self._BlobAsFile(plc_object, new_PLC_filename)
 
                 # Store new PLC filename based on md5 key
                 open(self._GetMD5FileName(), "w").write(md5sum)
 
                 # Then write the files
                 log = open(extra_files_log, "w")
-                for fname, fdata in extrafiles:
+                for fname, blobID in extrafiles:
                     fpath = os.path.join(self.workingdir, fname)
-                    open(fpath, "wb").write(fdata)
+                    self._BlobAsFile(blobID, fpath)
                     log.write(fname+'\n')
 
                 # Store new PLC filename
@@ -501,9 +550,7 @@
                 PLCprint(traceback.format_exc())
                 return False
 
-            if not replace_PLC_shared_object:
-                self.PLCStatus = PlcStatus.Stopped
-            elif self.LoadPLC():
+            if self.LoadPLC():
                 self.PLCStatus = PlcStatus.Stopped
             else:
                 self.PLCStatus = PlcStatus.Broken
--- a/runtime/Worker.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/runtime/Worker.py	Tue Dec 04 11:31:58 2018 +0100
@@ -9,6 +9,7 @@
 
 from __future__ import absolute_import
 import sys
+import six
 import thread
 from threading import Lock, Condition
 
--- a/targets/toolchain_gcc.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/targets/toolchain_gcc.py	Tue Dec 04 11:31:58 2018 +0100
@@ -72,23 +72,20 @@
         """
         return self.CTRInstance.GetTarget().getcontent().getLinker()
 
-    def GetBinaryCode(self):
-        try:
-            return open(self.exe_path, "rb").read()
-        except Exception:
-            return None
+    def GetBinaryPath(self):
+        return self.bin_path
 
     def _GetMD5FileName(self):
         return os.path.join(self.buildpath, "lastbuildPLC.md5")
 
-    def ResetBinaryCodeMD5(self):
+    def ResetBinaryMD5(self):
         self.md5key = None
         try:
             os.remove(self._GetMD5FileName())
         except Exception:
             pass
 
-    def GetBinaryCodeMD5(self):
+    def GetBinaryMD5(self):
         if self.md5key is not None:
             return self.md5key
         else:
@@ -100,8 +97,8 @@
     def SetBuildPath(self, buildpath):
         if self.buildpath != buildpath:
             self.buildpath = buildpath
-            self.exe = self.CTRInstance.GetProjectName() + self.extension
-            self.exe_path = os.path.join(self.buildpath, self.exe)
+            self.bin = self.CTRInstance.GetProjectName() + self.extension
+            self.bin_path = os.path.join(self.buildpath, self.bin)
             self.md5key = None
             self.srcmd5 = {}
 
@@ -152,9 +149,6 @@
                 wholesrcdata += self.concat_deps(CFileName)
         return hashlib.md5(wholesrcdata).hexdigest()
 
-    def calc_md5(self):
-        return hashlib.md5(self.GetBinaryCode()).hexdigest()
-
     def build(self):
         # Retrieve compiler and linker
         self.compiler = self.getCompiler()
@@ -165,7 +159,7 @@
         # ----------------- GENERATE OBJECT FILES ------------------------
         obns = []
         objs = []
-        relink = self.GetBinaryCode() is None
+        relink = not os.path.exists(self.bin_path)
         for Location, CFilesAndCFLAGS, _DoCalls in self.CTRInstance.LocationCFilesAndCFLAGS:
             if CFilesAndCFLAGS:
                 if Location:
@@ -213,14 +207,14 @@
 
             ALLldflags = ' '.join(self.getBuilderLDFLAGS())
 
-            self.CTRInstance.logger.write("   [CC]  " + ' '.join(obns)+" -> " + self.exe + "\n")
+            self.CTRInstance.logger.write("   [CC]  " + ' '.join(obns)+" -> " + self.bin + "\n")
 
             status, _result, _err_result = ProcessLogger(
                 self.CTRInstance.logger,
                 "\"%s\" %s -o \"%s\" %s" %
                 (self.linker,
                  listobjstring,
-                 self.exe_path,
+                 self.bin_path,
                  ALLldflags)
             ).spin()
 
@@ -228,10 +222,10 @@
                 return False
 
         else:
-            self.CTRInstance.logger.write("   [pass]  " + ' '.join(obns)+" -> " + self.exe + "\n")
+            self.CTRInstance.logger.write("   [pass]  " + ' '.join(obns)+" -> " + self.bin + "\n")
 
         # Calculate md5 key and get data for the new created PLC
-        self.md5key = self.calc_md5()
+        self.md5key = hashlib.md5(open(self.bin_path, "rb").read()).hexdigest()
 
         # Store new PLC filename based on md5 key
         f = open(self._GetMD5FileName(), "w")
--- a/targets/toolchain_makefile.py	Tue Nov 27 13:34:14 2018 +0100
+++ b/targets/toolchain_makefile.py	Tue Dec 04 11:31:58 2018 +0100
@@ -47,20 +47,20 @@
             self.buildpath = buildpath
             self.md5key = None
 
-    def GetBinaryCode(self):
+    def GetBinaryPath(self):
         return None
 
     def _GetMD5FileName(self):
         return os.path.join(self.buildpath, "lastbuildPLC.md5")
 
-    def ResetBinaryCodeMD5(self):
+    def ResetBinaryMD5(self):
         self.md5key = None
         try:
             os.remove(self._GetMD5FileName())
         except Exception:
             pass
 
-    def GetBinaryCodeMD5(self):
+    def GetBinaryMD5(self):
         if self.md5key is not None:
             return self.md5key
         else: