# HG changeset patch # User Edouard Tisserant # Date 1726230415 -7200 # Node ID 6337c9c2c379e0c1856cfe059b5280b90ddbaed2 # Parent fdbb3c4ebbf6047f79508a4c6b71bc376070d81f MQTT: add JSON payload corresponding to IEC Structured types - uses https://github.com/cesanta/frozen - frozen.c is added as a resulting c file in generat_C (fix toolchain_gcc.py as it was breaking dependency checking) - structured types are converted to C #define, in turn generating json_scanf and json_printf statements diff -r fdbb3c4ebbf6 -r 6337c9c2c379 mqtt/client.py --- a/mqtt/client.py Thu Sep 05 15:55:46 2024 +0200 +++ b/mqtt/client.py Fri Sep 13 14:26:55 2024 +0200 @@ -16,9 +16,12 @@ # and cmake build was invoked from this directory PahoMqttCLibraryPath = paths.ThirdPartyPath("paho.mqtt.c", "build", "src") -PahoMqttCIncludePaths = [ +frozen_path = paths.ThirdPartyPath("frozen") + +MqttCIncludePaths = [ paths.ThirdPartyPath("paho.mqtt.c", "build"), # VersionInfo.h - paths.ThirdPartyPath("paho.mqtt.c", "src") + paths.ThirdPartyPath("paho.mqtt.c", "src"), + frozen_path ] class MQTTClientEditor(ConfTreeNodeEditor): @@ -94,6 +97,10 @@ datatype_candidates = self.GetCTRoot().GetDataTypes() return datatype_candidates + def GetDataTypeInfos(self, typename): + tagname = "D::"+typename + return self.GetCTRoot().GetDataTypeInfos(tagname) + def GetConfig(self): def cfg(path): try: @@ -150,7 +157,7 @@ #include "beremiz.h" """ config = self.GetConfig() - c_code += self.modeldata.GenerateC(c_path, locstr, config) + c_code += self.modeldata.GenerateC(c_path, locstr, config, self.GetDataTypeInfos) with open(c_path, 'w') as c_file: c_file.write(c_code) @@ -164,9 +171,12 @@ LDFLAGS = [' "' + os.path.join(PahoMqttCLibraryPath, static_lib) + '"'] + libs - CFLAGS = ' '.join(['-I"' + path + '"' for path in PahoMqttCIncludePaths]) - - return [(c_path, CFLAGS)], LDFLAGS, True + CFLAGS = ' '.join(['-I"' + path + '"' for path in MqttCIncludePaths]) + + # TODO: add frozen only if using JSON + frozen_c_path = os.path.join(frozen_path, "frozen.c") + + return [(c_path, CFLAGS), (frozen_c_path, CFLAGS)], LDFLAGS, True def GetVariableLocationTree(self): current_location = self.GetCurrentLocation() diff -r fdbb3c4ebbf6 -r 6337c9c2c379 mqtt/mqtt_client_gen.py --- a/mqtt/mqtt_client_gen.py Thu Sep 05 15:55:46 2024 +0200 +++ b/mqtt/mqtt_client_gen.py Fri Sep 13 14:26:55 2024 +0200 @@ -4,7 +4,7 @@ import csv import functools from threading import Thread -from collections import OrderedDict +from collections import OrderedDict as OD import wx import wx.dataview as dv @@ -336,12 +336,14 @@ for row in data: writer.writerow([direction] + row) - def GenerateC(self, path, locstr, config): + def GenerateC(self, path, locstr, config, datatype_info_getter): c_template_filepath = paths.AbsNeighbourFile(__file__, "mqtt_template.c") c_template_file = open(c_template_filepath , 'rb') c_template = c_template_file.read() c_template_file.close() + json_types = OD() + formatdict = dict( locstr = locstr, uri = config["URI"], @@ -353,7 +355,8 @@ init_pubsub = "", retrieve = "", publish = "", - publish_changes = "" + publish_changes = "", + json_decl = "" ) @@ -384,19 +387,22 @@ if iec_type in MQTT_IEC_types: C_type, iec_size_prefix = MQTT_IEC_types[iec_type] c_loc_name = "__Q" + iec_size_prefix + locstr + "_" + str(iec_number) + encoding = "SIMPLE" else: C_type = iec_type.upper(); c_loc_name = "__Q" + locstr + "_" + str(iec_number) + json_types.setdefault(iec_type,OD()).setdefault("OUTPUT",[]).append(c_loc_name) + encoding = "JSON" formatdict["decl"] += """ DECL_VAR({iec_type}, {C_type}, {c_loc_name})""".format(**locals()) formatdict["init_pubsub"] += """ - INIT_PUBLICATION({Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals()) + INIT_PUBLICATION({encoding}, {Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals()) formatdict["publish"] += """ WRITE_VALUE({c_loc_name}, {C_type})""".format(**locals()) formatdict["publish_changes"] += """ - PUBLISH_CHANGE({Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals()) + PUBLISH_CHANGE({encoding}, {Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals()) # inputs need to be sorted for bisection search for row in sorted(self["input"]): @@ -404,19 +410,80 @@ if iec_type in MQTT_IEC_types: C_type, iec_size_prefix = MQTT_IEC_types[iec_type] c_loc_name = "__I" + iec_size_prefix + locstr + "_" + str(iec_number) + init_topic_call = "INIT_TOPIC" else: C_type = iec_type.upper(); c_loc_name = "__I" + locstr + "_" + str(iec_number) + init_topic_call = "INIT_JSON_TOPIC" + json_types.setdefault(iec_type,OD()).setdefault("INPUT",[]).append(c_loc_name) formatdict["decl"] += """ DECL_VAR({iec_type}, {C_type}, {c_loc_name})""".format(**locals()) formatdict["topics"] += """ - INIT_TOPIC({Topic}, {iec_type}, {c_loc_name})""".format(**locals()) + {init_topic_call}({Topic}, {iec_type}, {c_loc_name})""".format(**locals()) formatdict["init_pubsub"] += """ INIT_SUBSCRIPTION({Topic}, {QoS})""".format(**locals()) formatdict["retrieve"] += """ READ_VALUE({c_loc_name}, {C_type})""".format(**locals()) + def recurseJsonTypes(datatype, basetypes): + basetypes.append(datatype) + # add derivated type first fo we can expect the list to be sorted + # with base types in last position + infos = datatype_info_getter(datatype) + for element in infos["elements"]: + field_datatype = element["Type"] + if field_datatype not in MQTT_IEC_types: + recurseJsonTypes(field_datatype, basetypes) + + print(json_types) + + # collect all type dependencies + basetypes=[] # use a list to keep them ordered + for iec_type,_instances in json_types.items(): + recurseJsonTypes(iec_type, basetypes) + + done_types = set() + # go backard to get most derivated type definition last + # so that CPP can always find base type deinition before + for iec_type in reversed(basetypes): + # avoid repeating type definition + if iec_type in done_types: + continue + done_types.add(iec_type) + + C_type = iec_type.upper() + json_decl = "#define TYPE_" + C_type + "(_P, _A) \\\n" + + infos = datatype_info_getter(iec_type) + + elements = infos["elements"] + last = len(elements) - 1 + for idx, element in enumerate(elements): + field_iec_type = element["Type"] + field_C_type = field_iec_type.upper() + field_name = element["Name"].upper() + if field_iec_type in MQTT_IEC_types: + decl_type = "SIMPLE" + else: + decl_type = "OBJECT" + + json_decl += " _P##_"+decl_type+"(" + field_C_type + ", " + field_name + ", _A)" + if idx != last: + json_decl += " _P##_separator \\" + else: + json_decl += "\n" + json_decl += "\n" + + + formatdict["json_decl"] += json_decl + + for iec_type, instances in json_types.items(): + C_type = iec_type.upper() + for direction, instance_list in instances.items(): + for c_loc_name in instance_list: + formatdict["json_decl"] += "DECL_JSON_"+direction+"("+C_type+", "+c_loc_name+")\n" + Ccode = c_template.format(**formatdict) return Ccode diff -r fdbb3c4ebbf6 -r 6337c9c2c379 mqtt/mqtt_template.c --- a/mqtt/mqtt_template.c Thu Sep 05 15:55:46 2024 +0200 +++ b/mqtt/mqtt_template.c Fri Sep 13 14:26:55 2024 +0200 @@ -5,10 +5,15 @@ #include #include #include +#include + +#include "frozen.h" #include "MQTTClient.h" #include "MQTTClientPersistence.h" +#include "POUS.h" + #define _Log(level, ...) \ {{ \ char mstr[256]; \ @@ -42,13 +47,62 @@ #define UNCHANGED 0 #define DECL_VAR(iec_type, C_type, c_loc_name) \ -static C_type PLC_##c_loc_name##_buf = 0; \ -static C_type MQTT_##c_loc_name##_buf = 0; \ +static C_type PLC_##c_loc_name##_buf; \ +static C_type MQTT_##c_loc_name##_buf; \ static int MQTT_##c_loc_name##_state = UNCHANGED; /* systematically published at init */ \ C_type *c_loc_name = &PLC_##c_loc_name##_buf; {decl} +/* JSON topic content encoding macros matching "json_decl" in substitution*/ + +#define format_BOOL "%B" +#define format_SINT "%hhd" +#define format_USINT "%uhhd" +#define format_INT "%hd" +#define format_UINT "%uhd" +#define format_DINT "%d" +#define format_UDINT "%ud" +#define format_LINT "%ld" +#define format_ULINT "%uld" +#define format_REAL "%f" +#define format_LREAL "%Lf" +#define format_STRING "%*s" + +#define format_separator ", " + +#define format_SIMPLE(C_type, name, _A) #name " : " format_##C_type +#define format_OBJECT(C_type, name, _A) #name " : {{ " TYPE_##C_type(format, _A) " }}" + +#define arg_separator , + +#define arg_SIMPLE(C_type, name, data_ptr) data_ptr->name +#define arg_OBJECT(C_type, name, data_ptr) TYPE_##C_type(arg, (&data_ptr->name)) + +#define DECL_JSON_INPUT(C_type, c_loc_name) \ +int json_parse_##c_loc_name(char *json, const int len, void *void_ptr) {{ \ + C_type *struct_ptr = (C_type *)void_ptr; \ + return json_scanf(json, len, "{{" TYPE_##C_type(format,) "}}", TYPE_##C_type(arg, struct_ptr)); \ +}} + +/* Pre-allocated json output buffer for json_printf */ +#define json_out_size 1<<12 // 4K +static char json_out_buf[json_out_size] = {{0,}}; +static int json_out_len = 0; + +#define DECL_JSON_OUTPUT(C_type, c_loc_name) \ +int json_gen_##c_loc_name(C_type *struct_ptr) {{ \ + struct json_out out = JSON_OUT_BUF(json_out_buf, json_out_size); \ + json_out_len = json_printf(&out, "{{" TYPE_##C_type(format,) "}}", TYPE_##C_type(arg, struct_ptr)); \ + if(json_out_len > json_out_size){{ \ + json_out_len = 0; \ + return -EOVERFLOW; \ + }} \ + return 0; \ +}} + +{json_decl} + static MQTTClient client; #ifdef USE_MQTT_5 static MQTTClient_connectOptions conn_opts = MQTTClient_connectOptions_initializer5; @@ -79,14 +133,23 @@ /* thread that handles publish and reconnection */ static pthread_t MQTT_thread; -#define INIT_TOPIC(topic, iec_type, c_loc_name) \ -{{#topic, &MQTT_##c_loc_name##_buf, &MQTT_##c_loc_name##_state, iec_type##_ENUM}}, +#define INIT_TOPIC(topic, iec_type, c_loc_name) \ +{{#topic, &MQTT_##c_loc_name##_buf, &MQTT_##c_loc_name##_state, 0, .vartype = iec_type##_ENUM}}, + +#define INIT_JSON_TOPIC(topic, iec_type, c_loc_name) \ +{{#topic, &MQTT_##c_loc_name##_buf, &MQTT_##c_loc_name##_state, 1, .json_parse_func=json_parse_##c_loc_name}}, + +typedef int (*json_parse_func_t)(char *json, int len, void *void_ptr); static struct {{ const char *topic; //null terminated topic string void *mqtt_pdata; // pointer to data from/for MQTT stack int *mqtt_pchanged; // pointer to changed flag - __IEC_types_enum vartype; + int is_json_type; + union {{ + __IEC_types_enum vartype; + json_parse_func_t json_parse_func; + }}; }} topics [] = {{ {topics} }}; @@ -135,6 +198,7 @@ int size = sizeof(topics) / sizeof(topics[0]); int high = size - 1; int mid; + int is_json_type; // bisect topic among subscribed topics while (low <= high) {{ @@ -159,9 +223,15 @@ goto exit; found: - if(__get_type_enum_size(topics[mid].vartype) == message->payloadlen){{ + + is_json_type = topics[mid].is_json_type; + if(is_json_type || __get_type_enum_size(topics[mid].vartype) == message->payloadlen){{ if (pthread_mutex_lock(&MQTT_retrieve_mutex) == 0){{ - memcpy(topics[mid].mqtt_pdata, (char*)message->payload, message->payloadlen); + if(is_json_type){{ + (topics[mid].json_parse_func)((char*)message->payload, message->payloadlen, topics[mid].mqtt_pdata); + }} else {{ + memcpy(topics[mid].mqtt_pdata, (char*)message->payload, message->payloadlen); + }} *topics[mid].mqtt_pchanged = 1; pthread_mutex_unlock(&MQTT_retrieve_mutex); }} @@ -179,7 +249,7 @@ LogInfo("MQTT Init no auth\n"); #define INIT_x509(Verify, KeyStore, TrustStore) \ - LogInfo("MQTT Init x509 with %s,%s\n", KeyStore, TrustStore) \ + LogInfo("MQTT Init x509 with %s,%s\n", KeyStore?KeyStore:"NULL", TrustStore?TrustStore:"NULL")\ ssl_opts.verify = Verify; \ ssl_opts.keyStore = KeyStore; \ ssl_opts.trustStore = TrustStore; \ @@ -220,21 +290,30 @@ #ifdef USE_MQTT_5 -#define _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \ - MQTTResponse response = MQTTClient_publish5(client, #Topic, sizeof(C_type), \ - &MQTT_##c_loc_name##_buf, QoS, Retained, NULL, NULL); \ +#define _PUBLISH(Topic, QoS, cstring_size, cstring_ptr, Retained) \ + MQTTResponse response = MQTTClient_publish5(client, #Topic, cstring_size, \ + cstring_ptr, QoS, Retained, NULL, NULL); \ rc = response.reasonCode; \ MQTTResponse_free(response); #else -#define _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \ - rc = MQTTClient_publish(client, #Topic, sizeof(C_type), \ - &PLC_##c_loc_name##_buf, QoS, Retained, NULL); +#define _PUBLISH(Topic, QoS, cstring_size, cstring_ptr, Retained) \ + rc = MQTTClient_publish(client, #Topic, cstring_size, \ + cstring_ptr, QoS, Retained, NULL); #endif -#define INIT_PUBLICATION(Topic, QoS, C_type, c_loc_name, Retained) \ +#define PUBLISH_SIMPLE(Topic, QoS, C_type, c_loc_name, Retained) \ + _PUBLISH(Topic, QoS, sizeof(C_type), &MQTT_##c_loc_name##_buf, Retained) + +#define PUBLISH_JSON(Topic, QoS, C_type, c_loc_name, Retained) \ + int res = json_gen_##c_loc_name(&MQTT_##c_loc_name##_buf); \ + if(res == 0) {{ \ + _PUBLISH(Topic, QoS, json_out_len, json_out_buf, Retained) \ + }} + +#define INIT_PUBLICATION(encoding, Topic, QoS, C_type, c_loc_name, Retained) \ {{ \ int rc; \ - _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \ + PUBLISH_##encoding(Topic, QoS, C_type, c_loc_name, Retained) \ if (rc != MQTTCLIENT_SUCCESS) \ {{ \ LogError("MQTT client failed to init publication of '%s', return code %d\n", #Topic, rc);\ @@ -242,11 +321,11 @@ }} \ }} -#define PUBLISH_CHANGE(Topic, QoS, C_type, c_loc_name, Retained) \ +#define PUBLISH_CHANGE(encoding, Topic, QoS, C_type, c_loc_name, Retained) \ if(MQTT_##c_loc_name##_state == CHANGED) \ {{ \ int rc; \ - _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \ + PUBLISH_##encoding(Topic, QoS, C_type, c_loc_name, Retained) \ if (rc != MQTTCLIENT_SUCCESS) \ {{ \ LogError("MQTT client failed to publish '%s', return code %d\n", #Topic, rc); \ @@ -399,7 +478,7 @@ #define WRITE_VALUE(c_loc_name, C_type) \ /* TODO care about endianess */ \ - if(MQTT_##c_loc_name##_buf != PLC_##c_loc_name##_buf){{ \ + if(memcmp(&MQTT_##c_loc_name##_buf, &PLC_##c_loc_name##_buf, sizeof(C_type))){{ \ MQTT_##c_loc_name##_buf = PLC_##c_loc_name##_buf; \ MQTT_##c_loc_name##_state = CHANGED; \ MQTT_any_pub_var_changed = 1; \ diff -r fdbb3c4ebbf6 -r 6337c9c2c379 targets/toolchain_gcc.py --- a/targets/toolchain_gcc.py Thu Sep 05 15:55:46 2024 +0200 +++ b/targets/toolchain_gcc.py Fri Sep 13 14:26:55 2024 +0200 @@ -149,6 +149,8 @@ oldhash, deps = self.srcmd5.get(bn, (None, [])) # read source src = os.path.join(self.buildpath, bn) + if not os.path.exists(src): + return False # compute new hash newhash = compute_file_md5(src) # compare