Chunk based transfer for PLC binary and extra files, and some collateral code refactoring.
--- a/ProjectController.py Tue Nov 27 13:34:14 2018 +0100
+++ b/ProjectController.py Tue Dec 04 11:31:58 2018 +0100
@@ -891,19 +891,6 @@
self._builder = targetclass(self)
return self._builder
- def ResetBuildMD5(self):
- builder = self.GetBuilder()
- if builder is not None:
- builder.ResetBinaryCodeMD5()
- self.EnableMethod("_Transfer", False)
-
- def GetLastBuildMD5(self):
- builder = self.GetBuilder()
- if builder is not None:
- return builder.GetBinaryCodeMD5()
- else:
- return None
-
#
#
# C CODE GENERATION METHODS
@@ -1131,7 +1118,6 @@
# If IEC code gen fail, bail out.
if not IECGenRes:
self.logger.write_error(_("PLC code generation failed !\n"))
- self.ResetBuildMD5()
return False
# Reset variable and program list that are parsed from
@@ -1147,7 +1133,6 @@
builder = self.GetBuilder()
if builder is None:
self.logger.write_error(_("Fatal : cannot get builder.\n"))
- self.ResetBuildMD5()
return False
# Build
@@ -1156,9 +1141,9 @@
self.logger.write_error(_("C Build failed.\n"))
return False
except Exception:
+ builder.ResetBinaryMD5()
self.logger.write_error(_("C Build crashed !\n"))
self.logger.write_error(traceback.format_exc())
- self.ResetBuildMD5()
return False
self.logger.write(_("Successfully built.\n"))
@@ -1178,7 +1163,6 @@
self.logger.write_error(
_("Runtime IO extensions C code generation failed !\n"))
self.logger.write_error(traceback.format_exc())
- self.ResetBuildMD5()
return False
# Generate C code and compilation params from liraries
@@ -1189,7 +1173,6 @@
self.logger.write_error(
_("Runtime library extensions C code generation failed !\n"))
self.logger.write_error(traceback.format_exc())
- self.ResetBuildMD5()
return False
self.LocationCFilesAndCFLAGS = LibCFilesAndCFLAGS + \
@@ -1239,7 +1222,6 @@
except Exception:
self.logger.write_error(name + _(" generation failed !\n"))
self.logger.write_error(traceback.format_exc())
- self.ResetBuildMD5()
return False
self.logger.write(_("C code generated successfully.\n"))
return True
@@ -1820,26 +1802,18 @@
def CompareLocalAndRemotePLC(self):
if self._connector is None:
return
- # We are now connected. Update button status
- MD5 = self.GetLastBuildMD5()
+ builder = self.GetBuilder()
+ if builder is None :
+ return
+ MD5 = builder.GetBinaryMD5()
+ if MD5 is None:
+ return
# Check remote target PLC correspondance to that md5
- if MD5 is not None:
- if not self._connector.MatchMD5(MD5):
- # self.logger.write_warning(
- # _("Latest build does not match with target, please
- # transfer.\n"))
- self.EnableMethod("_Transfer", True)
- else:
- # self.logger.write(
- # _("Latest build matches target, no transfer needed.\n"))
- self.EnableMethod("_Transfer", True)
- # warns controller that program match
- self.ProgramTransferred()
- # self.EnableMethod("_Transfer", False)
+ if self._connector.MatchMD5(MD5):
+ self.ProgramTransferred()
else:
- # self.logger.write_warning(
- # _("Cannot compare latest build to target. Please build.\n"))
- self.EnableMethod("_Transfer", False)
+ self.logger.write(
+ _("Latest build does not match with connected target.\n"))
def _Disconnect(self):
self._SetConnector(None)
@@ -1855,8 +1829,13 @@
else:
return
- # Get the last build PLC's
- MD5 = self.GetLastBuildMD5()
+ builder = self.GetBuilder()
+ if builder is None:
+ self.logger.write_error(_("Fatal : cannot get builder.\n"))
+ return False
+
+ # recover md5 from last build
+ MD5 = builder.GetBinaryMD5()
# Check if md5 file is empty : ask user to build PLC
if MD5 is None:
@@ -1869,35 +1848,37 @@
self.logger.write(
_("Latest build already matches current target. Transfering anyway...\n"))
- # Get temprary directory path
+ # purge any non-finished transfer
+ # note: this would abord any runing transfer with error
+ self._connector.PurgeBlobs()
+
+ # transfer extra files
extrafiles = []
for extrafilespath in [self._getExtraFilesPath(),
self._getProjectFilesPath()]:
- extrafiles.extend(
- [(name, open(os.path.join(extrafilespath, name),
- 'rb').read())
- for name in os.listdir(extrafilespath)])
+ for name in os.listdir(extrafilespath):
+ extrafiles.append((
+ name,
+ self._connector.BlobFromFile(
+ os.path.join(extrafilespath, name))))
# Send PLC on target
- builder = self.GetBuilder()
- if builder is not None:
- data = builder.GetBinaryCode()
- if data is not None:
- if self._connector.NewPLC(MD5, data, extrafiles) and self.GetIECProgramsAndVariables():
- self.UnsubscribeAllDebugIECVariable()
- self.ProgramTransferred()
- if self.AppFrame is not None:
- self.AppFrame.CloseObsoleteDebugTabs()
- self.AppFrame.RefreshPouInstanceVariablesPanel()
- self.logger.write(_("Transfer completed successfully.\n"))
- self.AppFrame.LogViewer.ResetLogCounters()
- else:
- self.logger.write_error(_("Transfer failed\n"))
- self.HidePLCProgress()
- else:
- self.logger.write_error(
- _("No PLC to transfer (did build succeed ?)\n"))
+ object_path = builder.GetBinaryPath()
+ object_blob = self._connector.BlobFromFile(object_path)
+
+ if self._connector.NewPLC(MD5, object_blob, extrafiles):
+ self.ProgramTransferred()
+ self.AppFrame.CloseObsoleteDebugTabs()
+ self.AppFrame.LogViewer.ResetLogCounters()
+ if self.GetIECProgramsAndVariables():
+ self.UnsubscribeAllDebugIECVariable()
+ self.AppFrame.RefreshPouInstanceVariablesPanel()
+ self.logger.write(_("Transfer completed successfully.\n"))
+ else:
+ self.logger.write_error(_("Transfer failed\n"))
+
+ self.HidePLCProgress()
wx.CallAfter(self.UpdateMethodsFromPLCStatus)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/connectors/ConnectorBase.py Tue Dec 04 11:31:58 2018 +0100
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# See COPYING file for copyrights details.
+
+import md5
+
+class ConnectorBase(object):
+
+ chuncksize = 16384
+ def BlobFromFile(self, filepath):
+ s = md5.new()
+ blobID = s.digest() # empty md5, to support empty blob
+ with open(filepath, "rb") as f:
+ while True:
+ chunk = f.read(self.chuncksize)
+ if len(chunk) == 0: return blobID
+ blobID = self.AppendChunkToBlob(chunk, blobID)
+ s.update(chunk)
+ if blobID != s.digest(): return None
+
--- a/connectors/PYRO/__init__.py Tue Nov 27 13:34:14 2018 +0100
+++ b/connectors/PYRO/__init__.py Tue Dec 04 11:31:58 2018 +0100
@@ -150,4 +150,4 @@
self.__dict__[attrName] = member
return member
- return PyroProxyProxy()
+ return PyroProxyProxy
--- a/connectors/WAMP/__init__.py Tue Nov 27 13:34:14 2018 +0100
+++ b/connectors/WAMP/__init__.py Tue Dec 04 11:31:58 2018 +0100
@@ -156,10 +156,4 @@
# TODO : GetPLCID()
# TODO : PSK.UpdateID()
- # Try to get the proxy object
- try:
- return WampPLCObjectProxy()
- except Exception:
- confnodesroot.logger.write_error(_("WAMP connection to '%s' failed.\n") % location)
- confnodesroot.logger.write_error(traceback.format_exc())
- return None
+ return WampPLCObjectProxy
--- a/connectors/__init__.py Tue Nov 27 13:34:14 2018 +0100
+++ b/connectors/__init__.py Tue Dec 04 11:31:58 2018 +0100
@@ -29,6 +29,8 @@
from __future__ import absolute_import
from os import listdir, path
import util.paths as paths
+from connectors.ConnectorBase import ConnectorBase
+from types import ClassType
connectors_packages = ["PYRO","WAMP"]
@@ -64,8 +66,8 @@
Return a connector corresponding to the URI
or None if cannot connect to URI
"""
- scheme = uri.split("://")[0].upper()
- if scheme == "LOCAL":
+ _scheme = uri.split("://")[0].upper()
+ if _scheme == "LOCAL":
# Local is special case
# pyro connection to local runtime
# started on demand, listening on random port
@@ -73,17 +75,21 @@
runtime_port = confnodesroot.AppFrame.StartLocalRuntime(
taskbaricon=True)
uri = "PYROLOC://127.0.0.1:" + str(runtime_port)
- elif scheme in connectors:
- pass
- elif scheme[-1] == 'S' and scheme[:-1] in connectors:
- scheme = scheme[:-1]
+ elif _scheme in connectors:
+ scheme = _scheme
+ elif _scheme[-1] == 'S' and _scheme[:-1] in connectors:
+ scheme = _scheme[:-1]
else:
return None
- # import module according to uri type
- connectorclass = connectors[scheme]()
- return connectorclass(uri, confnodesroot)
+ # import module according to uri type and get connector specific baseclass
+ # first call to import the module,
+ # then call with parameters to create the class
+ connector_specific_class = connectors[scheme]()(uri, confnodesroot)
+ # new class inheriting from generic and specific connector base classes
+ return ClassType(_scheme + "_connector",
+ (ConnectorBase, connector_specific_class), {})()
def EditorClassFromScheme(scheme):
_Import_Dialogs()
--- a/runtime/PLCObject.py Tue Nov 27 13:34:14 2018 +0100
+++ b/runtime/PLCObject.py Tue Dec 04 11:31:58 2018 +0100
@@ -34,6 +34,9 @@
import Pyro.core as pyro
import six
from six.moves import _thread, xrange
+import md5
+from tempfile import mkstemp
+import shutil
from runtime.typemapping import TypeTranslator
from runtime.loglevels import LogLevelsDefault, LogLevelsCount
@@ -41,6 +44,8 @@
from runtime import PlcStatus
from runtime import MainWorker
+empty_md5_digest = md5.new().digest()
+
if os.name in ("nt", "ce"):
dlopen = _ctypes.LoadLibrary
dlclose = _ctypes.FreeLibrary
@@ -74,7 +79,11 @@
class PLCObject(object):
def __init__(self, WorkingDir, argv, statuschange, evaluator, pyruntimevars):
- self.workingdir = WorkingDir
+ self.workingdir = WorkingDir # must exits already
+ self.tmpdir = os.path.join(WorkingDir, 'tmp')
+ if os.path.exists(self.tmpdir):
+ shutil.rmtree(self.tmpdir)
+ os.mkdir(self.tmpdir)
# FIXME : is argv of any use nowadays ?
self.argv = [WorkingDir] + argv # force argv[0] to be "path" to exec...
self.statuschange = statuschange
@@ -91,6 +100,8 @@
self.TraceLock = Lock()
self.Traces = []
+ self._init_blobs()
+
# First task of worker -> no @RunInMain
def AutoLoad(self, autostart):
# Get the last transfered PLC
@@ -447,8 +458,52 @@
def GetPLCID(self):
return getPSKID()
- @RunInMain
- def NewPLC(self, md5sum, data, extrafiles):
+ def _init_blobs(self):
+ self.blobs = {}
+ if os.path.exists(self.tmpdir):
+ shutil.rmtree(self.tmpdir)
+ os.mkdir(self.tmpdir)
+
+ @RunInMain
+ def AppendChunkToBlob(self, data, blobID):
+ blob = ((mkstemp(dir=self.tmpdir) if data else None)\
+ + (md5.new(),)) \
+ if blobID == empty_md5_digest \
+ else self.blobs.pop(blobID, None)
+
+ if blob is None:
+ return None
+
+ fobj, path, md5sum = blob
+ md5sum.update(data)
+ newBlobID = md5sum.digest()
+ if data:
+ os.write(fobj,data)
+ self.blobs[newBlobID] = blob
+ return newBlobID
+
+ @RunInMain
+ def PurgeBlobs(self):
+ for fobj, path, md5sum in self.blobs:
+ os.close(fobj)
+ self._init_blobs()
+
+ def _BlobAsFile(self, blobID, newpath):
+ blob = self.blobs.pop(blobID, None)
+
+ if blob is None:
+ if blobID == md5.new().digest():
+ # create empty file
+ open(newpath,'r').close()
+ return
+ raise Exception(_("Missing data to create file: {}").format(newpath))
+
+ fobj, path, md5sum = blob
+ os.close(fobj)
+ shutil.move(path, newpath)
+
+ @RunInMain
+ def NewPLC(self, md5sum, plc_object, extrafiles):
if self.PLCStatus in [PlcStatus.Stopped, PlcStatus.Empty, PlcStatus.Broken]:
NewFileName = md5sum + lib_ext
extra_files_log = os.path.join(self.workingdir, "extra_files.txt")
@@ -458,18 +513,13 @@
else None
new_PLC_filename = os.path.join(self.workingdir, NewFileName)
- # Some platform (i.e. Xenomai) don't like reloading same .so file
- replace_PLC_shared_object = new_PLC_filename != old_PLC_filename
-
- if replace_PLC_shared_object:
- self.UnLoadPLC()
+ self.UnLoadPLC()
self.LogMessage("NewPLC (%s)" % md5sum)
self.PLCStatus = PlcStatus.Empty
try:
- if replace_PLC_shared_object:
- os.remove(old_PLC_filename)
+ os.remove(old_PLC_filename)
for filename in open(extra_files_log, "rt").readlines() + [extra_files_log]:
try:
os.remove(os.path.join(self.workingdir, filename.strip()))
@@ -480,17 +530,16 @@
try:
# Create new PLC file
- if replace_PLC_shared_object:
- open(new_PLC_filename, 'wb').write(data)
+ self._BlobAsFile(plc_object, new_PLC_filename)
# Store new PLC filename based on md5 key
open(self._GetMD5FileName(), "w").write(md5sum)
# Then write the files
log = open(extra_files_log, "w")
- for fname, fdata in extrafiles:
+ for fname, blobID in extrafiles:
fpath = os.path.join(self.workingdir, fname)
- open(fpath, "wb").write(fdata)
+ self._BlobAsFile(blobID, fpath)
log.write(fname+'\n')
# Store new PLC filename
@@ -501,9 +550,7 @@
PLCprint(traceback.format_exc())
return False
- if not replace_PLC_shared_object:
- self.PLCStatus = PlcStatus.Stopped
- elif self.LoadPLC():
+ if self.LoadPLC():
self.PLCStatus = PlcStatus.Stopped
else:
self.PLCStatus = PlcStatus.Broken
--- a/runtime/Worker.py Tue Nov 27 13:34:14 2018 +0100
+++ b/runtime/Worker.py Tue Dec 04 11:31:58 2018 +0100
@@ -9,6 +9,7 @@
from __future__ import absolute_import
import sys
+import six
import thread
from threading import Lock, Condition
--- a/targets/toolchain_gcc.py Tue Nov 27 13:34:14 2018 +0100
+++ b/targets/toolchain_gcc.py Tue Dec 04 11:31:58 2018 +0100
@@ -72,23 +72,20 @@
"""
return self.CTRInstance.GetTarget().getcontent().getLinker()
- def GetBinaryCode(self):
- try:
- return open(self.exe_path, "rb").read()
- except Exception:
- return None
+ def GetBinaryPath(self):
+ return self.bin_path
def _GetMD5FileName(self):
return os.path.join(self.buildpath, "lastbuildPLC.md5")
- def ResetBinaryCodeMD5(self):
+ def ResetBinaryMD5(self):
self.md5key = None
try:
os.remove(self._GetMD5FileName())
except Exception:
pass
- def GetBinaryCodeMD5(self):
+ def GetBinaryMD5(self):
if self.md5key is not None:
return self.md5key
else:
@@ -100,8 +97,8 @@
def SetBuildPath(self, buildpath):
if self.buildpath != buildpath:
self.buildpath = buildpath
- self.exe = self.CTRInstance.GetProjectName() + self.extension
- self.exe_path = os.path.join(self.buildpath, self.exe)
+ self.bin = self.CTRInstance.GetProjectName() + self.extension
+ self.bin_path = os.path.join(self.buildpath, self.bin)
self.md5key = None
self.srcmd5 = {}
@@ -152,9 +149,6 @@
wholesrcdata += self.concat_deps(CFileName)
return hashlib.md5(wholesrcdata).hexdigest()
- def calc_md5(self):
- return hashlib.md5(self.GetBinaryCode()).hexdigest()
-
def build(self):
# Retrieve compiler and linker
self.compiler = self.getCompiler()
@@ -165,7 +159,7 @@
# ----------------- GENERATE OBJECT FILES ------------------------
obns = []
objs = []
- relink = self.GetBinaryCode() is None
+ relink = not os.path.exists(self.bin_path)
for Location, CFilesAndCFLAGS, _DoCalls in self.CTRInstance.LocationCFilesAndCFLAGS:
if CFilesAndCFLAGS:
if Location:
@@ -213,14 +207,14 @@
ALLldflags = ' '.join(self.getBuilderLDFLAGS())
- self.CTRInstance.logger.write(" [CC] " + ' '.join(obns)+" -> " + self.exe + "\n")
+ self.CTRInstance.logger.write(" [CC] " + ' '.join(obns)+" -> " + self.bin + "\n")
status, _result, _err_result = ProcessLogger(
self.CTRInstance.logger,
"\"%s\" %s -o \"%s\" %s" %
(self.linker,
listobjstring,
- self.exe_path,
+ self.bin_path,
ALLldflags)
).spin()
@@ -228,10 +222,10 @@
return False
else:
- self.CTRInstance.logger.write(" [pass] " + ' '.join(obns)+" -> " + self.exe + "\n")
+ self.CTRInstance.logger.write(" [pass] " + ' '.join(obns)+" -> " + self.bin + "\n")
# Calculate md5 key and get data for the new created PLC
- self.md5key = self.calc_md5()
+ self.md5key = hashlib.md5(open(self.bin_path, "rb").read()).hexdigest()
# Store new PLC filename based on md5 key
f = open(self._GetMD5FileName(), "w")
--- a/targets/toolchain_makefile.py Tue Nov 27 13:34:14 2018 +0100
+++ b/targets/toolchain_makefile.py Tue Dec 04 11:31:58 2018 +0100
@@ -47,20 +47,20 @@
self.buildpath = buildpath
self.md5key = None
- def GetBinaryCode(self):
+ def GetBinaryPath(self):
return None
def _GetMD5FileName(self):
return os.path.join(self.buildpath, "lastbuildPLC.md5")
- def ResetBinaryCodeMD5(self):
+ def ResetBinaryMD5(self):
self.md5key = None
try:
os.remove(self._GetMD5FileName())
except Exception:
pass
- def GetBinaryCodeMD5(self):
+ def GetBinaryMD5(self):
if self.md5key is not None:
return self.md5key
else: