Reworked chunk based transfer to support duplicated files (i.e. files with same content, but different names)
--- a/ProjectController.py Fri Feb 01 14:14:13 2019 +0100
+++ b/ProjectController.py Wed Feb 13 09:41:35 2019 +0100
@@ -1873,11 +1873,14 @@
extrafiles.append((
name,
self._connector.BlobFromFile(
- os.path.join(extrafilespath, name))))
+ # use file name as a seed to avoid collisions
+ # with files having same content
+ os.path.join(extrafilespath, name),name)))
# Send PLC on target
object_path = builder.GetBinaryPath()
- object_blob = self._connector.BlobFromFile(object_path)
+ # arbitrarily use MD5 as a seed, could be any string
+ object_blob = self._connector.BlobFromFile(object_path, MD5)
self.HidePLCProgress()
--- a/connectors/ConnectorBase.py Fri Feb 01 14:14:13 2019 +0100
+++ b/connectors/ConnectorBase.py Wed Feb 13 09:41:35 2019 +0100
@@ -9,14 +9,14 @@
#chuncksize = 16384
chuncksize = 1024*1024
- def BlobFromFile(self, filepath):
+ def BlobFromFile(self, filepath, seed):
s = md5.new()
- blobID = s.digest() # empty md5, to support empty blob
+ s.update(seed)
+ blobID = self.SeedBlob(seed)
with open(filepath, "rb") as f:
- while True:
+ while blobID == s.digest():
chunk = f.read(self.chuncksize)
if len(chunk) == 0: return blobID
blobID = self.AppendChunkToBlob(chunk, blobID)
s.update(chunk)
- if blobID != s.digest(): return None
--- a/runtime/PLCObject.py Fri Feb 01 14:14:13 2019 +0100
+++ b/runtime/PLCObject.py Wed Feb 13 09:41:35 2019 +0100
@@ -45,8 +45,6 @@
from runtime import PlcStatus
from runtime import MainWorker
-empty_md5_digest = md5.new().digest()
-
if os.name in ("nt", "ce"):
dlopen = _ctypes.LoadLibrary
dlclose = _ctypes.FreeLibrary
@@ -468,11 +466,17 @@
os.mkdir(self.tmpdir)
@RunInMain
+ def SeedBlob(self, seed):
+ blob = (mkstemp(dir=self.tmpdir) + (md5.new(),))
+ fobj, path, md5sum = blob
+ md5sum.update(seed)
+ newBlobID = md5sum.digest()
+ self.blobs[newBlobID] = blob
+ return newBlobID
+
+ @RunInMain
def AppendChunkToBlob(self, data, blobID):
- blob = ((mkstemp(dir=self.tmpdir) if data else None)\
- + (md5.new(),)) \
- if blobID == empty_md5_digest \
- else self.blobs.pop(blobID, None)
+ blob = self.blobs.pop(blobID, None)
if blob is None:
return None
@@ -480,9 +484,8 @@
fobj, path, md5sum = blob
md5sum.update(data)
newBlobID = md5sum.digest()
- if data:
- os.write(fobj,data)
- self.blobs[newBlobID] = blob
+ os.write(fobj,data)
+ self.blobs[newBlobID] = blob
return newBlobID
@RunInMain
@@ -495,10 +498,6 @@
blob = self.blobs.pop(blobID, None)
if blob is None:
- if blobID == md5.new().digest():
- # create empty file
- open(newpath,'r').close()
- return
raise Exception(_("Missing data to create file: {}").format(newpath))
fobj, path, md5sum = blob
--- a/runtime/WampClient.py Fri Feb 01 14:14:13 2019 +0100
+++ b/runtime/WampClient.py Wed Feb 13 09:41:35 2019 +0100
@@ -57,6 +57,7 @@
("StopPLC", {}),
("GetPLCstatus", {}),
("GetPLCID", {}),
+ ("SeedBlob", {}),
("AppendChunkToBlob", {}),
("PurgeBlobs", {}),
("NewPLC", {}),