mqtt/mqtt_client_gen.py
author Edouard Tisserant <edouard@beremiz.fr>
Thu, 04 Jul 2024 11:24:04 +0200
changeset 3982 69d161fc2e08
parent 3981 74035ea6792c
child 3984 883a85b9ebcc
permissions -rw-r--r--
GCC BUILD: Prevent Memory Error when compiling huge projects.

Also deleted dead code.
from __future__ import print_function
from __future__ import absolute_import

import csv
import functools
from threading import Thread

import wx
import wx.dataview as dv

from perfect_hash import generate_code, IntSaltHash

MQTT_IEC_types = dict(
# IEC61131|  C  type   | sz
    BOOL  = ("uint8_t" , "X"),
    SINT  = ("int8_t"  , "B"),
    USINT = ("uint8_t" , "B"),
    INT   = ("int16_t" , "W"),
    UINT  = ("uint16_t", "W"),
    DINT  = ("uint32_t", "D"),
    UDINT = ("int32_t" , "D"),
    LINT  = ("int64_t" , "L"),
    ULINT = ("uint64_t", "L"),
    REAL  = ("float"   , "D"),
    LREAL = ("double"  , "L"),
)

"""
 QoS - Quality of Service
  0  : "At most one delivery"
  1  : "At least one delivery"
  2  : "Exactly one delivery"
"""
QoS_values = [0, 1, 2]

def boolean(v):
    if v in ["False","0"]:
        return False
    else:
        return bool(v)

lstcolnames  = [ "Topic",  "QoS",  "Retain", "Type", "Location"]
lstcolwidths = [     100,     50,       100,    100,         50]
lstcoltypess = [     str,    int,   boolean,    str,        int]
lstcoldeflts = [ "a/b/c",    "1",     False, "DINT",        "0"]
Location_column = lstcolnames.index("Location")

directions = ["input", "output"]

authParams = {
    "x509":[
        ("Certificate", "certificate.der"),
        ("PrivateKey", "private_key.pem")],
    "UserPassword":[
        ("User", None),
        ("Password", None)]}

class MQTTTopicListModel(dv.PyDataViewIndexListModel):
    def __init__(self, data, log):
        dv.PyDataViewIndexListModel.__init__(self, len(data))
        self.data = data
        self.log = log

    def GetColumnType(self, col):
        return "string"

    def GetValueByRow(self, row, col):
        return str(self.data[row][col])

    # This method is called when the user edits a data item in the view.
    def SetValueByRow(self, value, row, col):
        expectedtype = lstcoltypess[col]

        try:
            v = expectedtype(value)
        except ValueError: 
            self.log("String {} is invalid for type {}\n".format(value,expectedtype.__name__))
            return False

        if col == lstcolnames.index("QoS") and v not in QoS_values:
            self.log("{} is invalid for IdType\n".format(value))
            return False

        self.data[row][col] = v
        return True

    # Report how many columns this model provides data for.
    def GetColumnCount(self):
        return len(lstcolnames)

    # Report the number of rows in the model
    def GetCount(self):
        #self.log.write('GetCount')
        return len(self.data)

    # Called to check if non-standard attributes should be used in the
    # cell at (row, col)
    def GetAttrByRow(self, row, col, attr):
        if col == Location_column:
            attr.SetColour('blue')
            attr.SetBold(True)
            return True
        return False


    def DeleteRows(self, rows):
        # make a copy since we'll be sorting(mutating) the list
        # use reverse order so the indexes don't change as we remove items
        rows = sorted(rows, reverse=True)

        for row in rows:
            # remove it from our data structure
            del self.data[row]
            # notify the view(s) using this model that it has been removed
            self.RowDeleted(row)


    def AddRow(self, value):
        if self.data.append(value):
            # notify views
            self.RowAppended()

    def InsertDefaultRow(self, row):
        self.data.insert(row, lstcoldeflts[:])
        # notify views
        self.RowInserted(row)
    
    def ResetData(self):
        self.Reset(len(self.data))

class MQTTTopicListPanel(wx.Panel):
    def __init__(self, parent, log, model, direction):
        self.log = log
        wx.Panel.__init__(self, parent, -1)

        self.dvc = dv.DataViewCtrl(self,
                                   style=wx.BORDER_THEME
                                   | dv.DV_ROW_LINES
                                   | dv.DV_HORIZ_RULES
                                   | dv.DV_VERT_RULES
                                   | dv.DV_MULTIPLE
                                   )

        self.model = model

        self.dvc.AssociateModel(self.model)

        for idx,(colname,width) in enumerate(zip(lstcolnames,lstcolwidths)):
            self.dvc.AppendTextColumn(colname,  idx, width=width, mode=dv.DATAVIEW_CELL_EDITABLE)


        self.Sizer = wx.BoxSizer(wx.VERTICAL)

        self.direction =  direction
        titlestr = direction + " variables"

        title = wx.StaticText(self, label = titlestr)

        addbt = wx.Button(self, label="Add")
        self.Bind(wx.EVT_BUTTON, self.OnAddRow, addbt)
        delbt = wx.Button(self, label="Delete")
        self.Bind(wx.EVT_BUTTON, self.OnDeleteRows, delbt)

        topsizer = wx.BoxSizer(wx.HORIZONTAL)
        topsizer.Add(title, 1, wx.ALIGN_CENTER_VERTICAL|wx.LEFT|wx.RIGHT, 5)
        topsizer.Add(addbt, 0, wx.LEFT|wx.RIGHT, 5)
        topsizer.Add(delbt, 0, wx.LEFT|wx.RIGHT, 5)
        self.Sizer.Add(topsizer, 0, wx.EXPAND|wx.TOP|wx.BOTTOM, 5)
        self.Sizer.Add(self.dvc, 1, wx.EXPAND)


    def OnAddRow(self, evt):
        items = self.dvc.GetSelections()
        row = self.model.GetRow(items[0]) if items else 0
        self.model.InsertDefaultRow(row)

    def OnDeleteRows(self, evt):
        items = self.dvc.GetSelections()
        rows = [self.model.GetRow(item) for item in items]
        self.model.DeleteRows(rows)


class MQTTClientPanel(wx.Panel):
    def __init__(self, parent, modeldata, log, config_getter):
        self.log = log
        wx.Panel.__init__(self, parent)

        # TODO replace FlexGridSizer with a simpler one
        self.inout_sizer = wx.FlexGridSizer(cols=1, hgap=0, rows=2, vgap=0)
        self.inout_sizer.AddGrowableCol(0)
        self.inout_sizer.AddGrowableRow(0)

        self.config_getter = config_getter

        self.selected_splitter = wx.SplitterWindow(self, style=wx.SUNKEN_BORDER | wx.SP_3D)

        self.selected_datas = modeldata
        self.selected_models = { direction:MQTTTopicListModel(self.selected_datas[direction], log) for direction in directions }
        self.selected_lists = { direction:MQTTTopicListPanel(
                self.selected_splitter, log, 
                self.selected_models[direction], direction) 
            for direction in directions }

        self.selected_splitter.SplitHorizontally(*[self.selected_lists[direction] for direction in directions]+[300])

        self.inout_sizer.Add(self.selected_splitter, flag=wx.GROW)
        self.inout_sizer.Layout()
        self.SetAutoLayout(True)
        self.SetSizer(self.inout_sizer)

    def OnClose(self):
        pass

    def __del__(self):
        self.OnClose()

    def Reset(self):
        for direction in directions:
            self.selected_models[direction].ResetData() 
        

class MQTTClientList(list):
    def __init__(self, log, change_callback):
        super(MQTTClientList, self).__init__(self)
        self.log = log
        self.change_callback = change_callback

    def append(self, value):
        v = dict(list(zip(lstcolnames, value)))

        if type(v["Location"]) != int:
            if len(self) == 0:
                v["Location"] = 0
            else:
                iecnums = set(zip(*self)[Location_column])
                greatest = max(iecnums)
                holes = set(range(greatest)) - iecnums
                v["Location"] = min(holes) if holes else greatest+1

        if v["QoS"] not in QoS_values:
            self.log("Unknown QoS\n".format(value))
            return False

        try:
            for t,n in zip(lstcoltypess, lstcolnames):
                v[n] = t(v[n]) 
        except ValueError: 
            self.log("MQTT topic {} (Location={}) has invalid type\n".format(v["Topic"],v["Location"]))
            return False

        if len(self)>0 and v["Topic"] in list(zip(*self))[lstcolnames.index("Topic")]:
            self.log("MQTT topic {} (Location={}) already in the list\n".format(v["Topic"],v["Location"]))
            return False

        list.append(self, [v[n] for n in lstcolnames])

        self.change_callback()

        return True

    def __delitem__(self, index):
        list.__delitem__(self, index)
        self.change_callback()

class MQTTClientModel(dict):
    def __init__(self, log, change_callback = lambda : None):
        super(MQTTClientModel, self).__init__()
        for direction in directions:
            self[direction] = MQTTClientList(log, change_callback)

    def LoadCSV(self,path):
        with open(path, 'r') as csvfile:
            reader = csv.reader(csvfile, delimiter=',', quotechar='"')
            buf = {direction:[] for direction, _model in self.iteritems()}
            for direction, model in self.iteritems():
                self[direction][:] = []
            for row in reader:
                direction = row[0]
                # avoids calling change callback when loading CSV
                list.append(self[direction],row[1:])

    def SaveCSV(self,path):
        with open(path, 'w') as csvfile:
            for direction, data in self.items():
                writer = csv.writer(csvfile, delimiter=',',
                                quotechar='"', quoting=csv.QUOTE_MINIMAL)
                for row in data:
                    writer.writerow([direction] + row)

    def _TopicsPerfectHash(self, topics):
        template = """
#define NK  $NK       /* number of keys */
#define NG  $NG       /* number of vertices */
#define NS  $NS       /* length of array T1 and T2 */

int S1[] = {$S1};
int S2[] = {$S2};
int G[] = {$G};
char *K[] = {$K};
"""
		code = generate_code(topics, Hash=IntSaltHash, template=template)
        code += """
/* return index of key in K if key is found, -1 otherwise */
int MQTT_Topic_index(const char *key)
{
    int i, f1 = 0, f2 = 0;

    for (i = 0; key[i] != '\\0' && i < NS; i++) {
        f1 += S1[i] * key[i];
        f2 += S2[i] * key[i];
        f1 %= NG;
        f2 %= NG;
    }
    i = (G[f1] + G[f2]) % NG;
    if (i < NK && strcmp(key, K[i]) == 0)
        return i;

    return -1;
}
""" 
		return code

    def GenerateC(self, path, locstr, config):
        template = """/* code generated by beremiz MQTT extension */

#include "MQTTAsync.h"
#include "MQTTClientPersistence.h"

#define _Log(level, ...)                                                                           \\
    {{                                                                                             \\
        char mstr[256];                                                                            \\
        snprintf(mstr, 255, __VA_ARGS__);                                                          \\
        LogMessage(level, mstr, strlen(mstr));                                                     \\
    }}

#define LogInfo(...) _Log(LOG_INFO, __VA_ARGS__);
#define LogError(...) _Log(LOG_CRITICAL, __VA_ARGS__);
#define LogWarning(...) _Log(LOG_WARNING, __VA_ARGS__);

static inline void* loadFile(const char *const path) {{

    FILE *fp = fopen(path, "rb");
    if(!fp) {{
        errno = 0;
        LogError("MQTT could not open %s", path);
        return NULL;
    }}

    fseek(fp, 0, SEEK_END);
    size_t length = (size_t)ftell(fp);
    void* data = malloc(length);
    if(data) {{
        fseek(fp, 0, SEEK_SET);
        size_t read = fread(data, 1, fileContents.length, fp);
        if(read != length){{
            free(data);
            LogError("MQTT could not read %s", path);
        }}
    }} else {{
        LogError("MQTT Not enough memoty to load %s", path);
    }}
    fclose(fp);

    return data;
}}

static MQTTClient client;
static MQTTClient_connectOptions conn_opts = MQTTClient_connectOptions_initializer;

void trace_callback(enum MQTTASYNC_TRACE_LEVELS level, char* message)
{
	LogWarning("Paho MQTT Trace : %d, %s\n", level, message);
}

#define DECL_VAR(iec_type, C_type, c_loc_name)                                                       \\
static C_type c_loc_name##_buf = 0;                                                                 \\
C_type *c_loc_name = &c_loc_name##_buf;

{decl}

#define INIT_TOPIC(topic, iec_type, c_loc_name)                                                  \\
{topic, &c_loc_name##_buf, iec_type##_ENUM},

ststic struct {
	const char *topic; //null terminated topic string
	void *pdata; //pointer to data
	__IEC_types_enum vartype;
} topics [] = {
{topics}
}

void __cleanup_{locstr}(void)
{{
    MQTT_Client_disconnect(client);
    MQTT_Client_delete(client);
}}

#define INIT_NoAuth()                                                                              \\
    LogInfo("MQTT Init no auth");                                                                \\
    MQTT_ClientConfig_setDefault(cc);                                                                \\
    retval = MQTT_Client_connect(client, uri);

/* Note : Single policy is enforced here, by default open62541 client supports all policies */
#define INIT_x509(Policy, UpperCaseMode, PrivateKey, Certificate)                                  \\
    LogInfo("MQTT Init x509 %s,%s,%s,%s", #Policy, #UpperCaseMode, PrivateKey, Certificate);     \\
                                                                                                   \\
    MQTT_ByteString certificate = loadFile(Certificate);                                             \\
    MQTT_ByteString privateKey  = loadFile(PrivateKey);                                              \\
                                                                                                   \\
    cc->securityMode = MQTT_MESSAGESECURITYMODE_##UpperCaseMode;                                     \\
                                                                                                   \\
    /* replacement for default behaviour */                                                        \\
    /* MQTT_ClientConfig_setDefaultEncryption(cc, certificate, privateKey, NULL, 0, NULL, 0); */     \\
    do{{                                                                                           \\
        retval = MQTT_ClientConfig_setDefault(cc);                                                   \\
        if(retval != MQTT_STATUSCODE_GOOD)                                                           \\
            break;                                                                                 \\
                                                                                                   \\
        MQTT_SecurityPolicy *sp = (MQTT_SecurityPolicy*)                                               \\
            MQTT_realloc(cc->securityPolicies, sizeof(MQTT_SecurityPolicy) * 2);                       \\
        if(!sp){{                                                                                  \\
            retval = MQTT_STATUSCODE_BADOUTOFMEMORY;                                                 \\
            break;                                                                                 \\
        }}                                                                                         \\
        cc->securityPolicies = sp;                                                                 \\
                                                                                                   \\
        retval = MQTT_SecurityPolicy_##Policy(&cc->securityPolicies[cc->securityPoliciesSize],       \\
                                                 certificate, privateKey, &cc->logger);            \\
        if(retval != MQTT_STATUSCODE_GOOD) {{                                                        \\
            MQTT_LOG_WARNING(&cc->logger, MQTT_LOGCATEGORY_USERLAND,                                   \\
                           "Could not add SecurityPolicy Policy with error code %s",               \\
                           MQTT_StatusCode_name(retval));                                            \\
            MQTT_free(cc->securityPolicies);                                                         \\
            cc->securityPolicies = NULL;                                                           \\
            break;                                                                                 \\
        }}                                                                                         \\
                                                                                                   \\
        ++cc->securityPoliciesSize;                                                                \\
    }} while(0);                                                                                   \\
                                                                                                   \\
    retval = MQTT_Client_connect(client, uri);                                                       \\
                                                                                                   \\
    MQTT_ByteString_clear(&certificate);                                                             \\
    MQTT_ByteString_clear(&privateKey);

#define INIT_UserPassword(User, Password)                                                          \\
    LogInfo("MQTT Init UserPassword %s,%s", User, Password);                                     \\
    MQTT_ClientConfig_setDefault(cc);                                                                \\
    retval = MQTT_Client_connectUsername(client, uri, User, Password);

#define INIT_READ_VARIANT(ua_type, c_loc_name)                                                     \\
    MQTT_Variant_init(&c_loc_name##_variant);

#define INIT_WRITE_VARIANT(ua_type, ua_type_enum, c_loc_name)                                      \\
    MQTT_Variant_setScalar(&c_loc_name##_variant, (ua_type*)c_loc_name, &MQTT_TYPES[ua_type_enum]);

int __init_{locstr}(int argc,char **argv)
{{
    char *uri = "{uri}";
    char *clientID = "{clientID}";
    int rc;
    conn_opts = MQTTClient_connectOptions_initializer;

    if ((rc = MQTTClient_create(&client, uri, clientID,
        MQTTCLIENT_PERSISTENCE_NONE, NULL)) != MQTTCLIENT_SUCCESS)
    {
        printf("Failed to create client, return code %d\n", rc);
        rc = EXIT_FAILURE;
        goto exit;
    }

{init}

    if(retval != MQTT_STATUSCODE_GOOD) {{
        LogError("MQTT Init Failed %d", retval);
        MQTT_Client_delete(client);
        return EXIT_FAILURE;
    }}
    return 0;
}}

#define READ_VALUE(ua_type, ua_type_enum, c_loc_name, ua_nodeid_type, ua_nsidx, ua_node_id)        \\
    retval = MQTT_Client_readValueAttribute(                                                         \\
        client, ua_nodeid_type(ua_nsidx, ua_node_id), &c_loc_name##_variant);                      \\
    if(retval == MQTT_STATUSCODE_GOOD && MQTT_Variant_isScalar(&c_loc_name##_variant) &&               \\
       c_loc_name##_variant.type == &MQTT_TYPES[ua_type_enum]) {{                                    \\
            c_loc_name##_buf = *(ua_type*)c_loc_name##_variant.data;                               \\
            MQTT_Variant_clear(&c_loc_name##_variant);  /* Unalloc requiered on each read ! */       \\
    }}

void __retrieve_{locstr}(void)
{{
    MQTT_StatusCode retval;
{retrieve}
}}

#define WRITE_VALUE(ua_type, c_loc_name, ua_nodeid_type, ua_nsidx, ua_node_id)                     \\
    MQTT_Client_writeValueAttribute(                                                                 \\
        client, ua_nodeid_type(ua_nsidx, ua_node_id), &c_loc_name##_variant);

void __publish_{locstr}(void)
{{
{publish}
}}

"""
        
        formatdict = dict(
            locstr   = locstr,
            uri      = config["URI"],
            clientID = config["clientID"],
            decl     = "",
            topics   = "",
            cleanup  = "",
            init     = "",
            retrieve = "",
            publish  = "" 
        )

        AuthType = config["AuthType"]
        if AuthType == "x509":
            formatdict["init"] += """
    INIT_x509("{PrivateKey}", "{Certificate}")""".format(**config)
        elif AuthType == "UserPassword":
            formatdict["init"] += """
    INIT_UserPassword("{User}", "{Password}")""".format(**config)
        else:
            formatdict["init"] += """
    INIT_NoAuth()"""

		topics = OrderedDict()
        for direction, data in self.items():
            iec_direction_prefix = {"input": "__I", "output": "__Q"}[direction]
            for row in data:
				Topic, QoS, Retain, iec_type, iec_number = row
                C_type, iec_size_prefix = MQTT_IEC_types[iec_type]
                c_loc_name = iec_direction_prefix + iec_size_prefix + locstr + "_" + str(iec_number)

                formatdict["decl"] += """
DECL_VAR({iec_type}, {C_type}, {c_loc_name})""".format(**locals())

                formatdict["topics"] += """
INIT_TOPIC({Topic}, {iec_type}, {c_loc_name})""".format(**locals())
#
#                if direction == "input":
#                    formatdict["init"] += """
#    INIT_READ_VARIANT({ua_type}, {c_loc_name})""".format(**locals())
#                    formatdict["retrieve"] += """
#    READ_VALUE({ua_type}, {ua_type_enum}, {c_loc_name}, {ua_nodeid_type}, {ua_nsidx}, {ua_node_id})""".format(**locals())
#
#                if direction == "output":
#                    formatdict["init"] += """
#    INIT_WRITE_VARIANT({ua_type}, {ua_type_enum}, {c_loc_name})""".format(**locals())
#                    formatdict["publish"] += """
#    WRITE_VALUE({ua_type}, {c_loc_name}, {ua_nodeid_type}, {ua_nsidx}, {ua_node_id})""".format(**locals())

        Ccode = template.format(**formatdict)
        
        return Ccode

if __name__ == "__main__":

    import wx.lib.mixins.inspection as wit
    import sys,os

    app = wit.InspectableApp()

    frame = wx.Frame(None, -1, "MQTT Client Test App", size=(800,600))

    argc = len(sys.argv)

    config={}
    config["URI"] = sys.argv[1] if argc>1 else "tcp://localhost:1883"
    config["clientID"] = sys.argv[2] if argc>2 else ""
    config["AuthType"] = None

    if argc > 3:
        AuthType = sys.argv[3]
        config["AuthType"] = AuthType
        for (name, default), value in zip_longest(authParams[AuthType], sys.argv[4:]):
            if value is None:
                if default is None:
                    raise Exception(name+" param expected")
                value = default
            config[name] = value

    test_panel = wx.Panel(frame)
    test_sizer = wx.FlexGridSizer(cols=1, hgap=0, rows=2, vgap=0)
    test_sizer.AddGrowableCol(0)
    test_sizer.AddGrowableRow(0)

    modeldata = MQTTClientModel(print)

    mqtttestpanel = MQTTClientPanel(test_panel, modeldata, print, lambda:config)

    def OnGenerate(evt):
        dlg = wx.FileDialog(
            frame, message="Generate file as ...", defaultDir=os.getcwd(),
            defaultFile="", 
            wildcard="C (*.c)|*.c", style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT
            )

        if dlg.ShowModal() == wx.ID_OK:
            path = dlg.GetPath()
            Ccode = """
/*
In case open62541 was built just aside beremiz, you can build this test with:
gcc %s -o %s \\
    -I ../../open62541/plugins/include/ \\
    -I ../../open62541/build/src_generated/ \\
    -I ../../open62541/include/ \\
    -I ../../open62541/arch/ ../../open62541/build/bin/libopen62541.a
*/

"""%(path, path[:-2]) + modeldata.GenerateC(path, "test", config) + """

int LogMessage(uint8_t level, char* buf, uint32_t size){
    printf("log level:%d message:'%.*s'\\n", level, size, buf);
};

int main(int argc, char *argv[]) {

    __init_test(arc,argv);
   
    __retrieve_test();
   
    __publish_test();

    __cleanup_test();

    return EXIT_SUCCESS;
}
"""

            with open(path, 'w') as Cfile:
                Cfile.write(Ccode)


        dlg.Destroy()

    def OnLoad(evt):
        dlg = wx.FileDialog(
            frame, message="Choose a file",
            defaultDir=os.getcwd(),
            defaultFile="",
            wildcard="CSV (*.csv)|*.csv",
            style=wx.FD_OPEN | wx.FD_CHANGE_DIR | wx.FD_FILE_MUST_EXIST )

        if dlg.ShowModal() == wx.ID_OK:
            path = dlg.GetPath()
            modeldata.LoadCSV(path)
            mqtttestpanel.Reset()

        dlg.Destroy()

    def OnSave(evt):
        dlg = wx.FileDialog(
            frame, message="Save file as ...", defaultDir=os.getcwd(),
            defaultFile="", 
            wildcard="CSV (*.csv)|*.csv", style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT
            )

        if dlg.ShowModal() == wx.ID_OK:
            path = dlg.GetPath()
            modeldata.SaveCSV(path)

        dlg.Destroy()

    test_sizer.Add(mqtttestpanel, flag=wx.GROW|wx.EXPAND)

    testbt_sizer = wx.BoxSizer(wx.HORIZONTAL)

    loadbt = wx.Button(test_panel, label="Load")
    test_panel.Bind(wx.EVT_BUTTON, OnLoad, loadbt)

    savebt = wx.Button(test_panel, label="Save")
    test_panel.Bind(wx.EVT_BUTTON, OnSave, savebt)

    genbt = wx.Button(test_panel, label="Generate")
    test_panel.Bind(wx.EVT_BUTTON, OnGenerate, genbt)

    testbt_sizer.Add(loadbt, 0, wx.LEFT|wx.RIGHT, 5)
    testbt_sizer.Add(savebt, 0, wx.LEFT|wx.RIGHT, 5)
    testbt_sizer.Add(genbt, 0, wx.LEFT|wx.RIGHT, 5)

    test_sizer.Add(testbt_sizer, flag=wx.GROW)
    test_sizer.Layout()
    test_panel.SetAutoLayout(True)
    test_panel.SetSizer(test_sizer)

    def OnClose(evt):
        mqtttestpanel.OnClose()
        evt.Skip()

    frame.Bind(wx.EVT_CLOSE, OnClose)

    frame.Show()

    app.MainLoop()