# HG changeset patch # User Edouard Tisserant # Date 1623682119 -7200 # Node ID 095c73591b7e3c9c3978a9f5934b220407675664 # Parent 8d1cc99a8f545224eb8a22fa9985a0680120cc7f IDE: Cleaned up some useless tests in variable trace data handling code, changed from bare numpy arrays to RingBuffers inorder to avoid RAM outage and crash after long tracing session. diff -r 8d1cc99a8f54 -r 095c73591b7e ProjectController.py --- a/ProjectController.py Fri Jun 11 11:56:07 2021 +0200 +++ b/ProjectController.py Mon Jun 14 16:48:39 2021 +0200 @@ -40,6 +40,7 @@ from datetime import datetime from weakref import WeakKeyDictionary from functools import reduce +from itertools import izip from distutils.dir_util import copy_tree from six.moves import xrange @@ -1512,8 +1513,8 @@ for debug_tick, debug_buff in Traces: debug_vars = UnpackDebugBuffer( debug_buff, self.TracedIECTypes) - if debug_vars is not None and len(debug_vars) == len(self.TracedIECPath): - for IECPath, values_buffer, value in zip( + if debug_vars is not None: + for IECPath, values_buffer, value in izip( self.TracedIECPath, self.DebugValuesBuffers, debug_vars): @@ -1606,8 +1607,8 @@ WeakKeyDictionary(), # Callables [], # Data storage [(tick, data),...] "Registered", # Variable status - None, - buffer_list] # Forced value + None, # Forced value + buffer_list] self.IECdebug_datas[IECPath] = IECdebug_data else: IECdebug_data[4] |= buffer_list @@ -1681,27 +1682,28 @@ return self._connector.RemoteExec(script, **kwargs) def DispatchDebugValuesProc(self, event): + event.Skip() + start_time = time.time() self.debug_status, debug_ticks, buffers = self.SnapshotAndResetDebugValuesBuffers() - start_time = time.time() - if len(self.TracedIECPath) == len(buffers): - for IECPath, values in zip(self.TracedIECPath, buffers): - if len(values) > 0: - self.CallWeakcallables( - IECPath, "NewValues", debug_ticks, values) - if len(debug_ticks) > 0: - self.CallWeakcallables( - "__tick__", "NewDataAvailable", debug_ticks) if self.debug_status == PlcStatus.Broken: self.logger.write_warning( _("Debug: token rejected - other debug took over - reconnect to recover\n")) - else: - delay = time.time() - start_time - next_refresh = max(REFRESH_PERIOD - delay, 0.2 * delay) - if self.DispatchDebugValuesTimer is not None: - self.DispatchDebugValuesTimer.Start( - int(next_refresh * 1000), oneShot=True) - event.Skip() + return + + for IECPath, values in zip(self.TracedIECPath, buffers): + if len(values) > 0: + self.CallWeakcallables( + IECPath, "NewValues", debug_ticks, values) + if len(debug_ticks) > 0: + self.CallWeakcallables( + "__tick__", "NewDataAvailable", debug_ticks) + + delay = time.time() - start_time + next_refresh = max(REFRESH_PERIOD - delay, 0.2 * delay) + if self.DispatchDebugValuesTimer is not None: + res = self.DispatchDebugValuesTimer.Start( + int(next_refresh * 1000), oneShot=True) def KillDebugThread(self): if self.DispatchDebugValuesTimer is not None: diff -r 8d1cc99a8f54 -r 095c73591b7e controls/DebugVariablePanel/DebugVariableItem.py --- a/controls/DebugVariablePanel/DebugVariableItem.py Fri Jun 11 11:56:07 2021 +0200 +++ b/controls/DebugVariablePanel/DebugVariableItem.py Mon Jun 14 16:48:39 2021 +0200 @@ -26,8 +26,9 @@ from __future__ import absolute_import from datetime import timedelta import binascii -import numpy +import numpy as np from graphics.DebugDataConsumer import DebugDataConsumer, TYPE_TRANSLATOR +from controls.DebugVariablePanel.RingBuffer import RingBuffer # ------------------------------------------------------------------------------- # Constant for calculate CRC for string variables @@ -142,8 +143,8 @@ @return: Data as numpy.array([(tick, value, forced),...]) """ # Return immediately if data empty or none - if self.Data is None or len(self.Data) == 0: - return self.Data + if self.Data is None or self.Data.count == 0: + return None # Find nearest data outside given range indexes start_idx = (self.GetNearestData(start_tick, -1) @@ -154,7 +155,7 @@ else len(self.Data)) # Return data between indexes - return self.Data[start_idx:end_idx] + return self.Data.view[start_idx:end_idx] def GetRawValue(self, index): """ @@ -195,8 +196,8 @@ if len(values) > 0: # Return value range for data in given tick range return (data, - data[numpy.argmin(values), 1], - data[numpy.argmax(values), 1]) + data[np.argmin(values), 1], + data[np.argmax(values), 1]) # Return default values return data, None, None @@ -207,7 +208,7 @@ """ if self.StoreData and self.IsNumVariable(): # Init table storing data - self.Data = numpy.array([]).reshape(0, 3) + self.Data = RingBuffer(3) # Init table storing raw data if variable is strin self.RawData = ([] @@ -294,7 +295,7 @@ [float(tick), num_value, extra_value]) # Add New data to stored data table - self.Data = numpy.append(self.Data, data_values, axis=0) + self.Data.append(data_values) # Signal to debug variable panel to refresh self.Parent.HasNewData = True @@ -337,7 +338,7 @@ if tick is not None and self.Data is not None: # Return current value and forced flag if data empty - if len(self.Data) == 0: + if self.Data.count == 0: return self.Value, self.IsForced() # Get index of nearest data from tick given @@ -345,9 +346,9 @@ # Get value and forced flag at given index value, forced = \ - self.RawData[int(self.Data[idx, 2])] \ + self.RawData[int(self.Data.view[idx, 2])] \ if self.VariableType in ["STRING", "WSTRING"] \ - else self.Data[idx, 1:3] + else self.Data.view[idx, 1:3] if self.VariableType in ["TIME", "TOD", "DT", "DATE"]: value = timedelta(seconds=value) @@ -380,10 +381,10 @@ return None # Extract data ticks - ticks = self.Data[:, 0] + ticks = self.Data.view[:, 0] # Get nearest data from tick - idx = numpy.argmin(abs(ticks - tick)) + idx = np.searchsorted(ticks, tick) # Adjust data index according to constraint if adjust < 0 and ticks[idx] > tick and idx > 0 or \ diff -r 8d1cc99a8f54 -r 095c73591b7e controls/DebugVariablePanel/DebugVariablePanel.py --- a/controls/DebugVariablePanel/DebugVariablePanel.py Fri Jun 11 11:56:07 2021 +0200 +++ b/controls/DebugVariablePanel/DebugVariablePanel.py Mon Jun 14 16:48:39 2021 +0200 @@ -26,7 +26,7 @@ from __future__ import absolute_import from __future__ import division from functools import reduce -import numpy +import numpy as np import wx import wx.lib.buttons @@ -43,6 +43,7 @@ from controls.DebugVariablePanel.DebugVariableItem import DebugVariableItem from controls.DebugVariablePanel.DebugVariableTextViewer import DebugVariableTextViewer from controls.DebugVariablePanel.DebugVariableGraphicViewer import * +from controls.DebugVariablePanel.RingBuffer import RingBuffer MILLISECOND = 1000000 # Number of nanosecond in a millisecond @@ -205,7 +206,7 @@ main_sizer = wx.BoxSizer(wx.VERTICAL) - self.Ticks = numpy.array([]) # List of tick received + self.Ticks = RingBuffer() # List of tick received self.StartTick = 0 # Tick starting range of data displayed self.Fixed = False # Flag that range of data is fixed self.CursorTick = None # Tick of cursor for displaying values @@ -344,11 +345,11 @@ tick = ticks[-1] # Save tick as start tick for range if data is still empty - if len(self.Ticks) == 0: + if self.Ticks.count == 0: self.StartTick = ticks[0] # Add tick to list of ticks received - self.Ticks = numpy.append(self.Ticks, ticks) + self.Ticks.append(ticks) # Update start tick for range if range follow ticks received if not self.Fixed or tick < self.StartTick + self.CurrentRange: @@ -357,7 +358,7 @@ # Force refresh if graph is fixed because range of data received # is too small to fill data range selected if self.Fixed and \ - self.Ticks[-1] - self.Ticks[0] < self.CurrentRange: + self.Ticks.view[-1] - self.Ticks.view[0] < self.CurrentRange: self.Force = True self.HasNewData = False @@ -385,17 +386,16 @@ def MoveCursorTick(self, move): if self.CursorTick is not None: - cursor_tick = max(self.Ticks[0], - min(self.CursorTick + move, self.Ticks[-1])) - cursor_tick_idx = numpy.argmin(numpy.abs(self.Ticks - cursor_tick)) - if self.Ticks[cursor_tick_idx] == self.CursorTick: + cursor_tick = max(self.Ticks.view[0], + min(self.CursorTick + move, self.Ticks.view[-1])) + cursor_tick_idx = np.searchsorted(self.Ticks.view, cursor_tick) + if self.Ticks.view[cursor_tick_idx] == self.CursorTick: cursor_tick_idx = max(0, min(cursor_tick_idx + abs(move) // move, - len(self.Ticks) - 1)) - self.CursorTick = self.Ticks[cursor_tick_idx] + self.Ticks.count - 1)) + self.CursorTick = self.Ticks.view[cursor_tick_idx] self.StartTick = max( - self.Ticks[numpy.argmin( - numpy.abs(self.Ticks - self.CursorTick + self.CurrentRange))], + self.Ticks.view[np.searchsorted(self.Ticks.view, self.CursorTick + self.CurrentRange)], min(self.StartTick, self.CursorTick)) self.RefreshCanvasPosition() self.UpdateCursorTick() @@ -547,8 +547,8 @@ if self.CursorTick is not None: tick = self.CursorTick - elif len(self.Ticks) > 0: - tick = self.Ticks[-1] + elif self.Ticks.count > 0: + tick = self.Ticks.view[-1] else: tick = None if tick is not None: @@ -604,16 +604,16 @@ self.RefreshGraphicsSizer() def SetCanvasPosition(self, tick): - tick = max(self.Ticks[0], min(tick, self.Ticks[-1] - self.CurrentRange)) - self.StartTick = self.Ticks[numpy.argmin(numpy.abs(self.Ticks - tick))] + tick = max(self.Ticks.view[0], min(tick, self.Ticks.view[-1] - self.CurrentRange)) + self.StartTick = self.Ticks.view[np.searchsorted(self.Ticks.view, tick)] self.Fixed = True self.RefreshCanvasPosition() self.ForceRefresh() def RefreshCanvasPosition(self): - if len(self.Ticks) > 0: - pos = int(self.StartTick - self.Ticks[0]) - range = int(self.Ticks[-1] - self.Ticks[0]) + if len(self.Ticks.view) > 0: + pos = int(self.StartTick - self.Ticks.view[0]) + range = int(self.Ticks.view[-1] - self.Ticks.view[0]) else: pos = 0 range = 0 @@ -626,23 +626,23 @@ if new_range_idx != current_range_idx: self.CanvasRange.SetSelection(new_range_idx) self.CurrentRange = self.RANGE_VALUES[new_range_idx][1] / self.Ticktime - if len(self.Ticks) > 0: + if self.Ticks.count > 0: if tick is None: tick = self.StartTick + self.CurrentRange / 2. new_start_tick = min(tick - (tick - self.StartTick) * self.CurrentRange / current_range, - self.Ticks[-1] - self.CurrentRange) - self.StartTick = self.Ticks[numpy.argmin(numpy.abs(self.Ticks - new_start_tick))] - self.Fixed = new_start_tick < self.Ticks[-1] - self.CurrentRange + self.Ticks.view[-1] - self.CurrentRange) + self.StartTick = self.Ticks.view[np.searchsorted(self.Ticks.view, - new_start_tick)] + self.Fixed = new_start_tick < self.Ticks.view[-1] - self.CurrentRange self.ForceRefresh() def RefreshRange(self): - if len(self.Ticks) > 0: - if self.Fixed and self.Ticks[-1] - self.Ticks[0] < self.CurrentRange: + if self.Ticks.count > 0: + if self.Fixed and self.Ticks.view[-1] - self.Ticks.view[0] < self.CurrentRange: self.Fixed = False if self.Fixed: - self.StartTick = min(self.StartTick, self.Ticks[-1] - self.CurrentRange) - else: - self.StartTick = max(self.Ticks[0], self.Ticks[-1] - self.CurrentRange) + self.StartTick = min(self.StartTick, self.Ticks.view[-1] - self.CurrentRange) + else: + self.StartTick = max(self.Ticks.view[0], self.Ticks.view[-1] - self.CurrentRange) self.ForceRefresh() def OnRangeChanged(self, event): @@ -654,8 +654,8 @@ event.Skip() def OnCurrentButton(self, event): - if len(self.Ticks) > 0: - self.StartTick = max(self.Ticks[0], self.Ticks[-1] - self.CurrentRange) + if self.Ticks.count > 0: + self.StartTick = max(self.Ticks.view[0], self.Ticks.view[-1] - self.CurrentRange) self.ResetCursorTick() event.Skip() @@ -695,8 +695,8 @@ event.Skip() def OnPositionChanging(self, event): - if len(self.Ticks) > 0: - self.StartTick = self.Ticks[0] + event.GetPosition() + if self.Ticks.count > 0: + self.StartTick = self.Ticks.view[0] + event.GetPosition() self.Fixed = True self.ForceRefresh() event.Skip() @@ -908,7 +908,7 @@ self.ForceRefresh() def ResetGraphicsValues(self): - self.Ticks = numpy.array([]) + self.Ticks = RingBuffer() self.StartTick = 0 for panel in self.GraphicPanels: panel.ResetItemsData() diff -r 8d1cc99a8f54 -r 095c73591b7e controls/DebugVariablePanel/RingBuffer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/controls/DebugVariablePanel/RingBuffer.py Mon Jun 14 16:48:39 2021 +0200 @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This file is part of Beremiz +# Copyright (C) 2021: Edouard TISSERANT +# +# See COPYING file for copyrights details. + +# Based on Eelco Hoogendoorn stackoverflow answer about RingBuffer with numpy + +import numpy as np + + +class RingBuffer(object): + def __init__(self, width=None, size=65536, padding=None): + self.size = size + self.padding = size if padding is None else padding + shape = (self.size+self.padding,) + if width : + shape += (width,) + self.buffer = np.zeros(shape) + self.counter = 0 + self.full = False + + def append(self, data): + """this is an O(n) operation""" + data = data[-self.padding:] + n = len(data) + if self.remaining < n: self.compact() + self.buffer[self.counter+self.size:][:n] = data + self.counter += n + + @property + def count(self): + return self.counter if not self.full else self.size + + @property + def remaining(self): + return self.padding-self.counter + + @property + def view(self): + """this is always an O(1) operation""" + return self.buffer[self.counter:][:self.size] + + def compact(self): + """ + note: only when this function is called, is an O(size) performance hit incurred, + and this cost is amortized over the whole padding space + """ + print 'compacting' + self.buffer[:self.size] = self.view + self.counter = 0 + self.full = True + diff -r 8d1cc99a8f54 -r 095c73591b7e runtime/typemapping.py --- a/runtime/typemapping.py Fri Jun 11 11:56:07 2021 +0200 +++ b/runtime/typemapping.py Mon Jun 14 16:48:39 2021 +0200 @@ -92,7 +92,7 @@ buffoffset += sizeof(c_type) if iectype != "STRING" else len(value)+1 res.append(value) else: - break + return None if buffoffset and buffoffset == buffsize: return res return None