IDE: Cleaned up some useless tests in variable trace data handling code, changed from bare numpy arrays to RingBuffers inorder to avoid RAM outage and crash after long tracing session.
--- a/ProjectController.py Fri Jun 11 11:56:07 2021 +0200
+++ b/ProjectController.py Mon Jun 14 16:48:39 2021 +0200
@@ -40,6 +40,7 @@
from datetime import datetime
from weakref import WeakKeyDictionary
from functools import reduce
+from itertools import izip
from distutils.dir_util import copy_tree
from six.moves import xrange
@@ -1512,8 +1513,8 @@
for debug_tick, debug_buff in Traces:
debug_vars = UnpackDebugBuffer(
debug_buff, self.TracedIECTypes)
- if debug_vars is not None and len(debug_vars) == len(self.TracedIECPath):
- for IECPath, values_buffer, value in zip(
+ if debug_vars is not None:
+ for IECPath, values_buffer, value in izip(
self.TracedIECPath,
self.DebugValuesBuffers,
debug_vars):
@@ -1606,8 +1607,8 @@
WeakKeyDictionary(), # Callables
[], # Data storage [(tick, data),...]
"Registered", # Variable status
- None,
- buffer_list] # Forced value
+ None, # Forced value
+ buffer_list]
self.IECdebug_datas[IECPath] = IECdebug_data
else:
IECdebug_data[4] |= buffer_list
@@ -1681,27 +1682,28 @@
return self._connector.RemoteExec(script, **kwargs)
def DispatchDebugValuesProc(self, event):
+ event.Skip()
+ start_time = time.time()
self.debug_status, debug_ticks, buffers = self.SnapshotAndResetDebugValuesBuffers()
- start_time = time.time()
- if len(self.TracedIECPath) == len(buffers):
- for IECPath, values in zip(self.TracedIECPath, buffers):
- if len(values) > 0:
- self.CallWeakcallables(
- IECPath, "NewValues", debug_ticks, values)
- if len(debug_ticks) > 0:
- self.CallWeakcallables(
- "__tick__", "NewDataAvailable", debug_ticks)
if self.debug_status == PlcStatus.Broken:
self.logger.write_warning(
_("Debug: token rejected - other debug took over - reconnect to recover\n"))
- else:
- delay = time.time() - start_time
- next_refresh = max(REFRESH_PERIOD - delay, 0.2 * delay)
- if self.DispatchDebugValuesTimer is not None:
- self.DispatchDebugValuesTimer.Start(
- int(next_refresh * 1000), oneShot=True)
- event.Skip()
+ return
+
+ for IECPath, values in zip(self.TracedIECPath, buffers):
+ if len(values) > 0:
+ self.CallWeakcallables(
+ IECPath, "NewValues", debug_ticks, values)
+ if len(debug_ticks) > 0:
+ self.CallWeakcallables(
+ "__tick__", "NewDataAvailable", debug_ticks)
+
+ delay = time.time() - start_time
+ next_refresh = max(REFRESH_PERIOD - delay, 0.2 * delay)
+ if self.DispatchDebugValuesTimer is not None:
+ res = self.DispatchDebugValuesTimer.Start(
+ int(next_refresh * 1000), oneShot=True)
def KillDebugThread(self):
if self.DispatchDebugValuesTimer is not None:
--- a/controls/DebugVariablePanel/DebugVariableItem.py Fri Jun 11 11:56:07 2021 +0200
+++ b/controls/DebugVariablePanel/DebugVariableItem.py Mon Jun 14 16:48:39 2021 +0200
@@ -26,8 +26,9 @@
from __future__ import absolute_import
from datetime import timedelta
import binascii
-import numpy
+import numpy as np
from graphics.DebugDataConsumer import DebugDataConsumer, TYPE_TRANSLATOR
+from controls.DebugVariablePanel.RingBuffer import RingBuffer
# -------------------------------------------------------------------------------
# Constant for calculate CRC for string variables
@@ -142,8 +143,8 @@
@return: Data as numpy.array([(tick, value, forced),...])
"""
# Return immediately if data empty or none
- if self.Data is None or len(self.Data) == 0:
- return self.Data
+ if self.Data is None or self.Data.count == 0:
+ return None
# Find nearest data outside given range indexes
start_idx = (self.GetNearestData(start_tick, -1)
@@ -154,7 +155,7 @@
else len(self.Data))
# Return data between indexes
- return self.Data[start_idx:end_idx]
+ return self.Data.view[start_idx:end_idx]
def GetRawValue(self, index):
"""
@@ -195,8 +196,8 @@
if len(values) > 0:
# Return value range for data in given tick range
return (data,
- data[numpy.argmin(values), 1],
- data[numpy.argmax(values), 1])
+ data[np.argmin(values), 1],
+ data[np.argmax(values), 1])
# Return default values
return data, None, None
@@ -207,7 +208,7 @@
"""
if self.StoreData and self.IsNumVariable():
# Init table storing data
- self.Data = numpy.array([]).reshape(0, 3)
+ self.Data = RingBuffer(3)
# Init table storing raw data if variable is strin
self.RawData = ([]
@@ -294,7 +295,7 @@
[float(tick), num_value, extra_value])
# Add New data to stored data table
- self.Data = numpy.append(self.Data, data_values, axis=0)
+ self.Data.append(data_values)
# Signal to debug variable panel to refresh
self.Parent.HasNewData = True
@@ -337,7 +338,7 @@
if tick is not None and self.Data is not None:
# Return current value and forced flag if data empty
- if len(self.Data) == 0:
+ if self.Data.count == 0:
return self.Value, self.IsForced()
# Get index of nearest data from tick given
@@ -345,9 +346,9 @@
# Get value and forced flag at given index
value, forced = \
- self.RawData[int(self.Data[idx, 2])] \
+ self.RawData[int(self.Data.view[idx, 2])] \
if self.VariableType in ["STRING", "WSTRING"] \
- else self.Data[idx, 1:3]
+ else self.Data.view[idx, 1:3]
if self.VariableType in ["TIME", "TOD", "DT", "DATE"]:
value = timedelta(seconds=value)
@@ -380,10 +381,10 @@
return None
# Extract data ticks
- ticks = self.Data[:, 0]
+ ticks = self.Data.view[:, 0]
# Get nearest data from tick
- idx = numpy.argmin(abs(ticks - tick))
+ idx = np.searchsorted(ticks, tick)
# Adjust data index according to constraint
if adjust < 0 and ticks[idx] > tick and idx > 0 or \
--- a/controls/DebugVariablePanel/DebugVariablePanel.py Fri Jun 11 11:56:07 2021 +0200
+++ b/controls/DebugVariablePanel/DebugVariablePanel.py Mon Jun 14 16:48:39 2021 +0200
@@ -26,7 +26,7 @@
from __future__ import absolute_import
from __future__ import division
from functools import reduce
-import numpy
+import numpy as np
import wx
import wx.lib.buttons
@@ -43,6 +43,7 @@
from controls.DebugVariablePanel.DebugVariableItem import DebugVariableItem
from controls.DebugVariablePanel.DebugVariableTextViewer import DebugVariableTextViewer
from controls.DebugVariablePanel.DebugVariableGraphicViewer import *
+from controls.DebugVariablePanel.RingBuffer import RingBuffer
MILLISECOND = 1000000 # Number of nanosecond in a millisecond
@@ -205,7 +206,7 @@
main_sizer = wx.BoxSizer(wx.VERTICAL)
- self.Ticks = numpy.array([]) # List of tick received
+ self.Ticks = RingBuffer() # List of tick received
self.StartTick = 0 # Tick starting range of data displayed
self.Fixed = False # Flag that range of data is fixed
self.CursorTick = None # Tick of cursor for displaying values
@@ -344,11 +345,11 @@
tick = ticks[-1]
# Save tick as start tick for range if data is still empty
- if len(self.Ticks) == 0:
+ if self.Ticks.count == 0:
self.StartTick = ticks[0]
# Add tick to list of ticks received
- self.Ticks = numpy.append(self.Ticks, ticks)
+ self.Ticks.append(ticks)
# Update start tick for range if range follow ticks received
if not self.Fixed or tick < self.StartTick + self.CurrentRange:
@@ -357,7 +358,7 @@
# Force refresh if graph is fixed because range of data received
# is too small to fill data range selected
if self.Fixed and \
- self.Ticks[-1] - self.Ticks[0] < self.CurrentRange:
+ self.Ticks.view[-1] - self.Ticks.view[0] < self.CurrentRange:
self.Force = True
self.HasNewData = False
@@ -385,17 +386,16 @@
def MoveCursorTick(self, move):
if self.CursorTick is not None:
- cursor_tick = max(self.Ticks[0],
- min(self.CursorTick + move, self.Ticks[-1]))
- cursor_tick_idx = numpy.argmin(numpy.abs(self.Ticks - cursor_tick))
- if self.Ticks[cursor_tick_idx] == self.CursorTick:
+ cursor_tick = max(self.Ticks.view[0],
+ min(self.CursorTick + move, self.Ticks.view[-1]))
+ cursor_tick_idx = np.searchsorted(self.Ticks.view, cursor_tick)
+ if self.Ticks.view[cursor_tick_idx] == self.CursorTick:
cursor_tick_idx = max(0,
min(cursor_tick_idx + abs(move) // move,
- len(self.Ticks) - 1))
- self.CursorTick = self.Ticks[cursor_tick_idx]
+ self.Ticks.count - 1))
+ self.CursorTick = self.Ticks.view[cursor_tick_idx]
self.StartTick = max(
- self.Ticks[numpy.argmin(
- numpy.abs(self.Ticks - self.CursorTick + self.CurrentRange))],
+ self.Ticks.view[np.searchsorted(self.Ticks.view, self.CursorTick + self.CurrentRange)],
min(self.StartTick, self.CursorTick))
self.RefreshCanvasPosition()
self.UpdateCursorTick()
@@ -547,8 +547,8 @@
if self.CursorTick is not None:
tick = self.CursorTick
- elif len(self.Ticks) > 0:
- tick = self.Ticks[-1]
+ elif self.Ticks.count > 0:
+ tick = self.Ticks.view[-1]
else:
tick = None
if tick is not None:
@@ -604,16 +604,16 @@
self.RefreshGraphicsSizer()
def SetCanvasPosition(self, tick):
- tick = max(self.Ticks[0], min(tick, self.Ticks[-1] - self.CurrentRange))
- self.StartTick = self.Ticks[numpy.argmin(numpy.abs(self.Ticks - tick))]
+ tick = max(self.Ticks.view[0], min(tick, self.Ticks.view[-1] - self.CurrentRange))
+ self.StartTick = self.Ticks.view[np.searchsorted(self.Ticks.view, tick)]
self.Fixed = True
self.RefreshCanvasPosition()
self.ForceRefresh()
def RefreshCanvasPosition(self):
- if len(self.Ticks) > 0:
- pos = int(self.StartTick - self.Ticks[0])
- range = int(self.Ticks[-1] - self.Ticks[0])
+ if len(self.Ticks.view) > 0:
+ pos = int(self.StartTick - self.Ticks.view[0])
+ range = int(self.Ticks.view[-1] - self.Ticks.view[0])
else:
pos = 0
range = 0
@@ -626,23 +626,23 @@
if new_range_idx != current_range_idx:
self.CanvasRange.SetSelection(new_range_idx)
self.CurrentRange = self.RANGE_VALUES[new_range_idx][1] / self.Ticktime
- if len(self.Ticks) > 0:
+ if self.Ticks.count > 0:
if tick is None:
tick = self.StartTick + self.CurrentRange / 2.
new_start_tick = min(tick - (tick - self.StartTick) * self.CurrentRange / current_range,
- self.Ticks[-1] - self.CurrentRange)
- self.StartTick = self.Ticks[numpy.argmin(numpy.abs(self.Ticks - new_start_tick))]
- self.Fixed = new_start_tick < self.Ticks[-1] - self.CurrentRange
+ self.Ticks.view[-1] - self.CurrentRange)
+ self.StartTick = self.Ticks.view[np.searchsorted(self.Ticks.view, - new_start_tick)]
+ self.Fixed = new_start_tick < self.Ticks.view[-1] - self.CurrentRange
self.ForceRefresh()
def RefreshRange(self):
- if len(self.Ticks) > 0:
- if self.Fixed and self.Ticks[-1] - self.Ticks[0] < self.CurrentRange:
+ if self.Ticks.count > 0:
+ if self.Fixed and self.Ticks.view[-1] - self.Ticks.view[0] < self.CurrentRange:
self.Fixed = False
if self.Fixed:
- self.StartTick = min(self.StartTick, self.Ticks[-1] - self.CurrentRange)
- else:
- self.StartTick = max(self.Ticks[0], self.Ticks[-1] - self.CurrentRange)
+ self.StartTick = min(self.StartTick, self.Ticks.view[-1] - self.CurrentRange)
+ else:
+ self.StartTick = max(self.Ticks.view[0], self.Ticks.view[-1] - self.CurrentRange)
self.ForceRefresh()
def OnRangeChanged(self, event):
@@ -654,8 +654,8 @@
event.Skip()
def OnCurrentButton(self, event):
- if len(self.Ticks) > 0:
- self.StartTick = max(self.Ticks[0], self.Ticks[-1] - self.CurrentRange)
+ if self.Ticks.count > 0:
+ self.StartTick = max(self.Ticks.view[0], self.Ticks.view[-1] - self.CurrentRange)
self.ResetCursorTick()
event.Skip()
@@ -695,8 +695,8 @@
event.Skip()
def OnPositionChanging(self, event):
- if len(self.Ticks) > 0:
- self.StartTick = self.Ticks[0] + event.GetPosition()
+ if self.Ticks.count > 0:
+ self.StartTick = self.Ticks.view[0] + event.GetPosition()
self.Fixed = True
self.ForceRefresh()
event.Skip()
@@ -908,7 +908,7 @@
self.ForceRefresh()
def ResetGraphicsValues(self):
- self.Ticks = numpy.array([])
+ self.Ticks = RingBuffer()
self.StartTick = 0
for panel in self.GraphicPanels:
panel.ResetItemsData()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/controls/DebugVariablePanel/RingBuffer.py Mon Jun 14 16:48:39 2021 +0200
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This file is part of Beremiz
+# Copyright (C) 2021: Edouard TISSERANT
+#
+# See COPYING file for copyrights details.
+
+# Based on Eelco Hoogendoorn stackoverflow answer about RingBuffer with numpy
+
+import numpy as np
+
+
+class RingBuffer(object):
+ def __init__(self, width=None, size=65536, padding=None):
+ self.size = size
+ self.padding = size if padding is None else padding
+ shape = (self.size+self.padding,)
+ if width :
+ shape += (width,)
+ self.buffer = np.zeros(shape)
+ self.counter = 0
+ self.full = False
+
+ def append(self, data):
+ """this is an O(n) operation"""
+ data = data[-self.padding:]
+ n = len(data)
+ if self.remaining < n: self.compact()
+ self.buffer[self.counter+self.size:][:n] = data
+ self.counter += n
+
+ @property
+ def count(self):
+ return self.counter if not self.full else self.size
+
+ @property
+ def remaining(self):
+ return self.padding-self.counter
+
+ @property
+ def view(self):
+ """this is always an O(1) operation"""
+ return self.buffer[self.counter:][:self.size]
+
+ def compact(self):
+ """
+ note: only when this function is called, is an O(size) performance hit incurred,
+ and this cost is amortized over the whole padding space
+ """
+ print 'compacting'
+ self.buffer[:self.size] = self.view
+ self.counter = 0
+ self.full = True
+
--- a/runtime/typemapping.py Fri Jun 11 11:56:07 2021 +0200
+++ b/runtime/typemapping.py Mon Jun 14 16:48:39 2021 +0200
@@ -92,7 +92,7 @@
buffoffset += sizeof(c_type) if iectype != "STRING" else len(value)+1
res.append(value)
else:
- break
+ return None
if buffoffset and buffoffset == buffsize:
return res
return None