# HG changeset patch # User Edouard Tisserant # Date 1543919518 -3600 # Node ID 8742337a9fe3a6af0b9c4a0f143126865e084dd8 # Parent ed6b0e905fcb9e7b1421ecd1292043875c567e40 Chunk based transfer for PLC binary and extra files, and some collateral code refactoring. diff -r ed6b0e905fcb -r 8742337a9fe3 ProjectController.py --- a/ProjectController.py Tue Nov 27 13:34:14 2018 +0100 +++ b/ProjectController.py Tue Dec 04 11:31:58 2018 +0100 @@ -891,19 +891,6 @@ self._builder = targetclass(self) return self._builder - def ResetBuildMD5(self): - builder = self.GetBuilder() - if builder is not None: - builder.ResetBinaryCodeMD5() - self.EnableMethod("_Transfer", False) - - def GetLastBuildMD5(self): - builder = self.GetBuilder() - if builder is not None: - return builder.GetBinaryCodeMD5() - else: - return None - # # # C CODE GENERATION METHODS @@ -1131,7 +1118,6 @@ # If IEC code gen fail, bail out. if not IECGenRes: self.logger.write_error(_("PLC code generation failed !\n")) - self.ResetBuildMD5() return False # Reset variable and program list that are parsed from @@ -1147,7 +1133,6 @@ builder = self.GetBuilder() if builder is None: self.logger.write_error(_("Fatal : cannot get builder.\n")) - self.ResetBuildMD5() return False # Build @@ -1156,9 +1141,9 @@ self.logger.write_error(_("C Build failed.\n")) return False except Exception: + builder.ResetBinaryMD5() self.logger.write_error(_("C Build crashed !\n")) self.logger.write_error(traceback.format_exc()) - self.ResetBuildMD5() return False self.logger.write(_("Successfully built.\n")) @@ -1178,7 +1163,6 @@ self.logger.write_error( _("Runtime IO extensions C code generation failed !\n")) self.logger.write_error(traceback.format_exc()) - self.ResetBuildMD5() return False # Generate C code and compilation params from liraries @@ -1189,7 +1173,6 @@ self.logger.write_error( _("Runtime library extensions C code generation failed !\n")) self.logger.write_error(traceback.format_exc()) - self.ResetBuildMD5() return False self.LocationCFilesAndCFLAGS = LibCFilesAndCFLAGS + \ @@ -1239,7 +1222,6 @@ except Exception: self.logger.write_error(name + _(" generation failed !\n")) self.logger.write_error(traceback.format_exc()) - self.ResetBuildMD5() return False self.logger.write(_("C code generated successfully.\n")) return True @@ -1820,26 +1802,18 @@ def CompareLocalAndRemotePLC(self): if self._connector is None: return - # We are now connected. Update button status - MD5 = self.GetLastBuildMD5() + builder = self.GetBuilder() + if builder is None : + return + MD5 = builder.GetBinaryMD5() + if MD5 is None: + return # Check remote target PLC correspondance to that md5 - if MD5 is not None: - if not self._connector.MatchMD5(MD5): - # self.logger.write_warning( - # _("Latest build does not match with target, please - # transfer.\n")) - self.EnableMethod("_Transfer", True) - else: - # self.logger.write( - # _("Latest build matches target, no transfer needed.\n")) - self.EnableMethod("_Transfer", True) - # warns controller that program match - self.ProgramTransferred() - # self.EnableMethod("_Transfer", False) + if self._connector.MatchMD5(MD5): + self.ProgramTransferred() else: - # self.logger.write_warning( - # _("Cannot compare latest build to target. Please build.\n")) - self.EnableMethod("_Transfer", False) + self.logger.write( + _("Latest build does not match with connected target.\n")) def _Disconnect(self): self._SetConnector(None) @@ -1855,8 +1829,13 @@ else: return - # Get the last build PLC's - MD5 = self.GetLastBuildMD5() + builder = self.GetBuilder() + if builder is None: + self.logger.write_error(_("Fatal : cannot get builder.\n")) + return False + + # recover md5 from last build + MD5 = builder.GetBinaryMD5() # Check if md5 file is empty : ask user to build PLC if MD5 is None: @@ -1869,35 +1848,37 @@ self.logger.write( _("Latest build already matches current target. Transfering anyway...\n")) - # Get temprary directory path + # purge any non-finished transfer + # note: this would abord any runing transfer with error + self._connector.PurgeBlobs() + + # transfer extra files extrafiles = [] for extrafilespath in [self._getExtraFilesPath(), self._getProjectFilesPath()]: - extrafiles.extend( - [(name, open(os.path.join(extrafilespath, name), - 'rb').read()) - for name in os.listdir(extrafilespath)]) + for name in os.listdir(extrafilespath): + extrafiles.append(( + name, + self._connector.BlobFromFile( + os.path.join(extrafilespath, name)))) # Send PLC on target - builder = self.GetBuilder() - if builder is not None: - data = builder.GetBinaryCode() - if data is not None: - if self._connector.NewPLC(MD5, data, extrafiles) and self.GetIECProgramsAndVariables(): - self.UnsubscribeAllDebugIECVariable() - self.ProgramTransferred() - if self.AppFrame is not None: - self.AppFrame.CloseObsoleteDebugTabs() - self.AppFrame.RefreshPouInstanceVariablesPanel() - self.logger.write(_("Transfer completed successfully.\n")) - self.AppFrame.LogViewer.ResetLogCounters() - else: - self.logger.write_error(_("Transfer failed\n")) - self.HidePLCProgress() - else: - self.logger.write_error( - _("No PLC to transfer (did build succeed ?)\n")) + object_path = builder.GetBinaryPath() + object_blob = self._connector.BlobFromFile(object_path) + + if self._connector.NewPLC(MD5, object_blob, extrafiles): + self.ProgramTransferred() + self.AppFrame.CloseObsoleteDebugTabs() + self.AppFrame.LogViewer.ResetLogCounters() + if self.GetIECProgramsAndVariables(): + self.UnsubscribeAllDebugIECVariable() + self.AppFrame.RefreshPouInstanceVariablesPanel() + self.logger.write(_("Transfer completed successfully.\n")) + else: + self.logger.write_error(_("Transfer failed\n")) + + self.HidePLCProgress() wx.CallAfter(self.UpdateMethodsFromPLCStatus) diff -r ed6b0e905fcb -r 8742337a9fe3 connectors/ConnectorBase.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/connectors/ConnectorBase.py Tue Dec 04 11:31:58 2018 +0100 @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# See COPYING file for copyrights details. + +import md5 + +class ConnectorBase(object): + + chuncksize = 16384 + def BlobFromFile(self, filepath): + s = md5.new() + blobID = s.digest() # empty md5, to support empty blob + with open(filepath, "rb") as f: + while True: + chunk = f.read(self.chuncksize) + if len(chunk) == 0: return blobID + blobID = self.AppendChunkToBlob(chunk, blobID) + s.update(chunk) + if blobID != s.digest(): return None + diff -r ed6b0e905fcb -r 8742337a9fe3 connectors/PYRO/__init__.py --- a/connectors/PYRO/__init__.py Tue Nov 27 13:34:14 2018 +0100 +++ b/connectors/PYRO/__init__.py Tue Dec 04 11:31:58 2018 +0100 @@ -150,4 +150,4 @@ self.__dict__[attrName] = member return member - return PyroProxyProxy() + return PyroProxyProxy diff -r ed6b0e905fcb -r 8742337a9fe3 connectors/WAMP/__init__.py --- a/connectors/WAMP/__init__.py Tue Nov 27 13:34:14 2018 +0100 +++ b/connectors/WAMP/__init__.py Tue Dec 04 11:31:58 2018 +0100 @@ -156,10 +156,4 @@ # TODO : GetPLCID() # TODO : PSK.UpdateID() - # Try to get the proxy object - try: - return WampPLCObjectProxy() - except Exception: - confnodesroot.logger.write_error(_("WAMP connection to '%s' failed.\n") % location) - confnodesroot.logger.write_error(traceback.format_exc()) - return None + return WampPLCObjectProxy diff -r ed6b0e905fcb -r 8742337a9fe3 connectors/__init__.py --- a/connectors/__init__.py Tue Nov 27 13:34:14 2018 +0100 +++ b/connectors/__init__.py Tue Dec 04 11:31:58 2018 +0100 @@ -29,6 +29,8 @@ from __future__ import absolute_import from os import listdir, path import util.paths as paths +from connectors.ConnectorBase import ConnectorBase +from types import ClassType connectors_packages = ["PYRO","WAMP"] @@ -64,8 +66,8 @@ Return a connector corresponding to the URI or None if cannot connect to URI """ - scheme = uri.split("://")[0].upper() - if scheme == "LOCAL": + _scheme = uri.split("://")[0].upper() + if _scheme == "LOCAL": # Local is special case # pyro connection to local runtime # started on demand, listening on random port @@ -73,17 +75,21 @@ runtime_port = confnodesroot.AppFrame.StartLocalRuntime( taskbaricon=True) uri = "PYROLOC://127.0.0.1:" + str(runtime_port) - elif scheme in connectors: - pass - elif scheme[-1] == 'S' and scheme[:-1] in connectors: - scheme = scheme[:-1] + elif _scheme in connectors: + scheme = _scheme + elif _scheme[-1] == 'S' and _scheme[:-1] in connectors: + scheme = _scheme[:-1] else: return None - # import module according to uri type - connectorclass = connectors[scheme]() - return connectorclass(uri, confnodesroot) + # import module according to uri type and get connector specific baseclass + # first call to import the module, + # then call with parameters to create the class + connector_specific_class = connectors[scheme]()(uri, confnodesroot) + # new class inheriting from generic and specific connector base classes + return ClassType(_scheme + "_connector", + (ConnectorBase, connector_specific_class), {})() def EditorClassFromScheme(scheme): _Import_Dialogs() diff -r ed6b0e905fcb -r 8742337a9fe3 runtime/PLCObject.py --- a/runtime/PLCObject.py Tue Nov 27 13:34:14 2018 +0100 +++ b/runtime/PLCObject.py Tue Dec 04 11:31:58 2018 +0100 @@ -34,6 +34,9 @@ import Pyro.core as pyro import six from six.moves import _thread, xrange +import md5 +from tempfile import mkstemp +import shutil from runtime.typemapping import TypeTranslator from runtime.loglevels import LogLevelsDefault, LogLevelsCount @@ -41,6 +44,8 @@ from runtime import PlcStatus from runtime import MainWorker +empty_md5_digest = md5.new().digest() + if os.name in ("nt", "ce"): dlopen = _ctypes.LoadLibrary dlclose = _ctypes.FreeLibrary @@ -74,7 +79,11 @@ class PLCObject(object): def __init__(self, WorkingDir, argv, statuschange, evaluator, pyruntimevars): - self.workingdir = WorkingDir + self.workingdir = WorkingDir # must exits already + self.tmpdir = os.path.join(WorkingDir, 'tmp') + if os.path.exists(self.tmpdir): + shutil.rmtree(self.tmpdir) + os.mkdir(self.tmpdir) # FIXME : is argv of any use nowadays ? self.argv = [WorkingDir] + argv # force argv[0] to be "path" to exec... self.statuschange = statuschange @@ -91,6 +100,8 @@ self.TraceLock = Lock() self.Traces = [] + self._init_blobs() + # First task of worker -> no @RunInMain def AutoLoad(self, autostart): # Get the last transfered PLC @@ -447,8 +458,52 @@ def GetPLCID(self): return getPSKID() - @RunInMain - def NewPLC(self, md5sum, data, extrafiles): + def _init_blobs(self): + self.blobs = {} + if os.path.exists(self.tmpdir): + shutil.rmtree(self.tmpdir) + os.mkdir(self.tmpdir) + + @RunInMain + def AppendChunkToBlob(self, data, blobID): + blob = ((mkstemp(dir=self.tmpdir) if data else None)\ + + (md5.new(),)) \ + if blobID == empty_md5_digest \ + else self.blobs.pop(blobID, None) + + if blob is None: + return None + + fobj, path, md5sum = blob + md5sum.update(data) + newBlobID = md5sum.digest() + if data: + os.write(fobj,data) + self.blobs[newBlobID] = blob + return newBlobID + + @RunInMain + def PurgeBlobs(self): + for fobj, path, md5sum in self.blobs: + os.close(fobj) + self._init_blobs() + + def _BlobAsFile(self, blobID, newpath): + blob = self.blobs.pop(blobID, None) + + if blob is None: + if blobID == md5.new().digest(): + # create empty file + open(newpath,'r').close() + return + raise Exception(_("Missing data to create file: {}").format(newpath)) + + fobj, path, md5sum = blob + os.close(fobj) + shutil.move(path, newpath) + + @RunInMain + def NewPLC(self, md5sum, plc_object, extrafiles): if self.PLCStatus in [PlcStatus.Stopped, PlcStatus.Empty, PlcStatus.Broken]: NewFileName = md5sum + lib_ext extra_files_log = os.path.join(self.workingdir, "extra_files.txt") @@ -458,18 +513,13 @@ else None new_PLC_filename = os.path.join(self.workingdir, NewFileName) - # Some platform (i.e. Xenomai) don't like reloading same .so file - replace_PLC_shared_object = new_PLC_filename != old_PLC_filename - - if replace_PLC_shared_object: - self.UnLoadPLC() + self.UnLoadPLC() self.LogMessage("NewPLC (%s)" % md5sum) self.PLCStatus = PlcStatus.Empty try: - if replace_PLC_shared_object: - os.remove(old_PLC_filename) + os.remove(old_PLC_filename) for filename in open(extra_files_log, "rt").readlines() + [extra_files_log]: try: os.remove(os.path.join(self.workingdir, filename.strip())) @@ -480,17 +530,16 @@ try: # Create new PLC file - if replace_PLC_shared_object: - open(new_PLC_filename, 'wb').write(data) + self._BlobAsFile(plc_object, new_PLC_filename) # Store new PLC filename based on md5 key open(self._GetMD5FileName(), "w").write(md5sum) # Then write the files log = open(extra_files_log, "w") - for fname, fdata in extrafiles: + for fname, blobID in extrafiles: fpath = os.path.join(self.workingdir, fname) - open(fpath, "wb").write(fdata) + self._BlobAsFile(blobID, fpath) log.write(fname+'\n') # Store new PLC filename @@ -501,9 +550,7 @@ PLCprint(traceback.format_exc()) return False - if not replace_PLC_shared_object: - self.PLCStatus = PlcStatus.Stopped - elif self.LoadPLC(): + if self.LoadPLC(): self.PLCStatus = PlcStatus.Stopped else: self.PLCStatus = PlcStatus.Broken diff -r ed6b0e905fcb -r 8742337a9fe3 runtime/Worker.py --- a/runtime/Worker.py Tue Nov 27 13:34:14 2018 +0100 +++ b/runtime/Worker.py Tue Dec 04 11:31:58 2018 +0100 @@ -9,6 +9,7 @@ from __future__ import absolute_import import sys +import six import thread from threading import Lock, Condition diff -r ed6b0e905fcb -r 8742337a9fe3 targets/toolchain_gcc.py --- a/targets/toolchain_gcc.py Tue Nov 27 13:34:14 2018 +0100 +++ b/targets/toolchain_gcc.py Tue Dec 04 11:31:58 2018 +0100 @@ -72,23 +72,20 @@ """ return self.CTRInstance.GetTarget().getcontent().getLinker() - def GetBinaryCode(self): - try: - return open(self.exe_path, "rb").read() - except Exception: - return None + def GetBinaryPath(self): + return self.bin_path def _GetMD5FileName(self): return os.path.join(self.buildpath, "lastbuildPLC.md5") - def ResetBinaryCodeMD5(self): + def ResetBinaryMD5(self): self.md5key = None try: os.remove(self._GetMD5FileName()) except Exception: pass - def GetBinaryCodeMD5(self): + def GetBinaryMD5(self): if self.md5key is not None: return self.md5key else: @@ -100,8 +97,8 @@ def SetBuildPath(self, buildpath): if self.buildpath != buildpath: self.buildpath = buildpath - self.exe = self.CTRInstance.GetProjectName() + self.extension - self.exe_path = os.path.join(self.buildpath, self.exe) + self.bin = self.CTRInstance.GetProjectName() + self.extension + self.bin_path = os.path.join(self.buildpath, self.bin) self.md5key = None self.srcmd5 = {} @@ -152,9 +149,6 @@ wholesrcdata += self.concat_deps(CFileName) return hashlib.md5(wholesrcdata).hexdigest() - def calc_md5(self): - return hashlib.md5(self.GetBinaryCode()).hexdigest() - def build(self): # Retrieve compiler and linker self.compiler = self.getCompiler() @@ -165,7 +159,7 @@ # ----------------- GENERATE OBJECT FILES ------------------------ obns = [] objs = [] - relink = self.GetBinaryCode() is None + relink = not os.path.exists(self.bin_path) for Location, CFilesAndCFLAGS, _DoCalls in self.CTRInstance.LocationCFilesAndCFLAGS: if CFilesAndCFLAGS: if Location: @@ -213,14 +207,14 @@ ALLldflags = ' '.join(self.getBuilderLDFLAGS()) - self.CTRInstance.logger.write(" [CC] " + ' '.join(obns)+" -> " + self.exe + "\n") + self.CTRInstance.logger.write(" [CC] " + ' '.join(obns)+" -> " + self.bin + "\n") status, _result, _err_result = ProcessLogger( self.CTRInstance.logger, "\"%s\" %s -o \"%s\" %s" % (self.linker, listobjstring, - self.exe_path, + self.bin_path, ALLldflags) ).spin() @@ -228,10 +222,10 @@ return False else: - self.CTRInstance.logger.write(" [pass] " + ' '.join(obns)+" -> " + self.exe + "\n") + self.CTRInstance.logger.write(" [pass] " + ' '.join(obns)+" -> " + self.bin + "\n") # Calculate md5 key and get data for the new created PLC - self.md5key = self.calc_md5() + self.md5key = hashlib.md5(open(self.bin_path, "rb").read()).hexdigest() # Store new PLC filename based on md5 key f = open(self._GetMD5FileName(), "w") diff -r ed6b0e905fcb -r 8742337a9fe3 targets/toolchain_makefile.py --- a/targets/toolchain_makefile.py Tue Nov 27 13:34:14 2018 +0100 +++ b/targets/toolchain_makefile.py Tue Dec 04 11:31:58 2018 +0100 @@ -47,20 +47,20 @@ self.buildpath = buildpath self.md5key = None - def GetBinaryCode(self): + def GetBinaryPath(self): return None def _GetMD5FileName(self): return os.path.join(self.buildpath, "lastbuildPLC.md5") - def ResetBinaryCodeMD5(self): + def ResetBinaryMD5(self): self.md5key = None try: os.remove(self._GetMD5FileName()) except Exception: pass - def GetBinaryCodeMD5(self): + def GetBinaryMD5(self): if self.md5key is not None: return self.md5key else: