8  *  the Free Software Foundation, either version 3 of the License, or

     8  *  the Free Software Foundation, either version 3 of thest_whitespaceLicense, or

   359  * The state machine has 7 possible states (INITIAL, config, decl, body, st, il, sfc)

   359  * There is a main state machine...

   360  * 

   361  *       +---> INITIAL <-------> config

   362  *       |        \

   363  *       |        V

   364  *       |   header_state

   365  *       |        |

   366  *       |        V

   367  *     vardecl_list_state <------> var_decl

   368  *       ^        | 

   369  *       |        | [using push()]

   370  *       |        |

   371  *       |        V

   372  *       |       body, 

   373  *       |        |

   374  *       |        | 

   375  *       |   -------------------

   376  *       |   |       |         |

   377  *       |   v       v         v

   378  *       |  st      il        sfc

   379  *       |   |       |         |  [using pop() when leaving st/il/sfc => goes to vardecl_list_state]

   380  *       |   |       |         |

   381  *       -----------------------

   382  *

   383  * NOTE:- When inside sfc, and an action or transition in ST/IL is found, then 

   384  *        we also push() to the body state. This means that sometimes, when pop()ing

   385  *        from st and il, the state machine may return to the sfc state!

   386  *      - The transitions form sfc to body will be decided by bison, which will

   387  *        tell flex to do the transition by calling cmd_goto_body_state().

   388  *   

   389  * 

   372  *   decl_state    -> push(decl_state); goto(body_state)

   393  * 

   373  *                     (when the last END_VAR is found, i.e. the function body starts)

   394  *   INITIAL -> goto(header_state)

   374  *   decl_state    -> push(decl_state); goto(sfc_state)

   395  *               (when a FUNCTION, FUNCTION_BLOCK, or PROGRAM is found)

   396  *   header_state -> goto(vardecl_list_state)

   397  *               (When the first VAR token is found, i.e. at begining of first VAR .. END_VAR declaration)

   398  * 

   399  *  vardecl_list_state -> push current state (vardecl_list_state), and goto(vardecl_state) 

   400  *                (when a VAR token is found)

   401  *   vardecl_state -> pop() to (vardecl_list_state) 

   402  *                (when a END_VAR token is found)

   403  * 

   404  *   vardecl_list_state -> push current state (vardecl_list_state), and goto(body_state) 

   405  *                (when the last END_VAR is found!)

   406  *

   407  *   body_state    -> goto(sfc_state)

   380  *   st_state      -> pop()

   413  *   st_state      -> pop() to vardecl_list_state

   383  *   il_state      -> pop()

   416  *   il_state      -> pop() to vardecl_list_state

   386  *   decl_state    -> goto(INITIAL)

   419  *   sfc_state     -> pop() to vardecl_list_state

   388  *   sfc_state     -> goto(INITIAL)

   421  * 

   422  *   vardecl_list_state -> goto(INITIAL)

   392  *   sfc_state     -> push(sfc_state); goto(body_state)

   426  * 

   427  *  

   428  *   sfc_state     -> push current state(sfc_state); goto(body_state)

   394  *   sfc_state     -> push(sfc_state); goto(sfc_qualifier_state)

   430  *   sfc_state     -> push current state(sfc_state); goto(sfc_qualifier_state)

   396  *   sfc_qualifier_state -> pop()

   432  *   sfc_qualifier_state -> pop() to sfc_state

   416 /* we are parsing a function, program or function block declaration */

   456 /* we are looking for the first VAR inside a function's, program's or function block's declaration */

   417 %s decl_state

   457 /* This is not exclusive (%x) as we must be able to parse the identifier and data types of a function/FB */

   418 

   458 %s header_state

   419 /* we will be parsing a function body. Whether il/st/sfc remains to be determined */

   459 

   460 /* we are parsing a function, program or function block sequence of VAR..END_VAR delcarations */

   461 %x vardecl_list_state 

   462 /* a substate of the vardecl_list_state: we are inside a specific VAR .. END_VAR */

   463 %s vardecl_state

   464 

   465 /* we will be parsing a function body/action/transition. Whether il/st/sfc remains to be determined */

   458 file_include_pragma_beg		"{#include"{st_whitespace_only}\"

   507 file_include_pragma_beg		"{#include"{st_whitespace}\"

   459 file_include_pragma_end		\"{st_whitespace_only}"}"

   508 file_include_pragma_end		\"{st_whitespace}"}"

   524 

   575 pragma ("{"[^}]*"}")|("{{"([^}]|"}"[^}])*"}}")

   525 pragma "{"[^}]*"}"|"{{"([^}]|"}"[^}])*"}}"

   576 

   526 

   577 

   578 

   579 /* COMMENTS */

   580 /* ======== */

   581 

   582 /* In order to allow nested comments, comments are handled by a specific comment_state state */

   583 /* Whenever a "(*" is found, we push the current state onto the stack, and enter a new instance of the comment_state state.

   584  * Whenever a "*)" is found, we pop a state off the stack

   585  */

   586 

   587 /* comments... */

   588 comment_beg  "(*"

   589 comment_end  "*)"

   590 

   591 /* However, bison has a shift/reduce conflict in bison, when parsing formal function/FB

   592  * invocations with the 'NOT <variable_name> =>' syntax (which needs two look ahead 

   593  * tokens to be parsed correctly - and bison being LALR(1) only supports one).

   594  * The current work around requires flex to completely parse the '<variable_name> =>'

   595  * sequence. This sequence includes whitespace and/or comments between the 

   596  * <variable_name> and the "=>" token.

   597  * 

   598  * This flex rule (sendto_identifier_token) uses the whitespace/comment as trailing context,

   599  * which means we can not use the comment_state method of specifying/finding and ignoring 

   600  * comments.

   601  * 

   602  * For this reason only, we must also define what a complete comment looks like, so

   603  * it may be used in this rule. Since the rule uses the whitespace_or_comment

   604  * construct as trailing context, this definition of comment must not use any

   605  * trailing context either.

   606  * 

   607  * Aditionally, it is not possible to define nested comments in flex without the use of

   608  * states, so for this particular location, we do NOT support nested comments.

   609  */

   537 comment_text		{not_asterisk}|(({asterisk}+){not_close_parenthesis_nor_asterisk})

   620 comment_text	({not_asterisk})|(({asterisk}+){not_close_parenthesis_nor_asterisk})

   538 

   543 3.1 Whitespace

   628  * Whitespace is clearly defined (see IEC 61131-3 v2, section 2.1.4)

   544  (NOTE: Whitespace IS clearly defined, to include newline!!! See section 2.1.4!!!)

   629  * 

   545  No definition of whitespace is given, in other words, the characters that may be used to seperate language tokens are not pecisely defined. One may nevertheless make an inteligent guess of using the space (' '), and other characters also commonly considered whitespace in other programming languages (horizontal tab, vertical tab, form feed, etc.).

   630  * Whitespace definition includes the newline character.

   546  The main question is whether the newline character should be considered whitespace. IL language statements use an EOL token (End Of Line) to distinguish between some language constructs. The EOL token itself is openly defined as "normally consist[ing] of the 'paragraph separator' ", leaving the final choice open to each implemention. If we choose the newline character to represent the EOL token, it may then not be considered whitespace.

   631  * 

   547  On the other hand, some examples that come in a non-normative annex of the specification allow function declarations to span multiple3.1 Whitespace

   632  * However, the standard is inconsistent in that in IL the newline character 

   548  (NOTE: Whitespace IS clearly defined, to include newline!!! See section 2.1.4!!!)

   633  * is considered a token (EOL - end of line). 

   549  No definition of whitespace is given, in other words, the characters that may be used to seperate language tokens are not pecisely defined. One may nevertheless make an inteligent guess of using the space (' '), and other characters also commonly considered whitespace in other programming languages (horizontal tab, vertical tab, form feed, etc.).

   634  * In our implementation we therefore have two definitions of whitespace

   550  The main question is whether the newline character should be considered whitespace. IL language statements use an EOL token (End Of Line) to distinguish between some language constructs. The EOL token itself is openly defined as "normally consist[ing] of the 'paragraph separator' ", leaving the final choice open to each implemention. If we choose the newline character to represent the EOL token, it may then not be considered whitespace.

   635  *   - one for ST, that includes the newline character

   551  On the other hand, some examples that come in a non-normative annex of the specification allow function declarations to span multiple lines, which means that the newline character is being considered as whitespace.

   636  *   - one for IL without the newline character.

   552  Our implementation works around this issue by including the new line character in the whitespace while parsing function declarations and the ST language, and parsing it as the EOL token only while parsing IL language statements. This requires the use of a state machine in the lexical parser that needs at least some knowledge of the syntax itself.

   637  * Additionally, when parsing IL, the newline character is treated as the EOL token.

   553 */

   638  * This requires the use of a state machine in the lexical parser that needs at least 

   554 /* NOTE: Our definition of whitespace will only work in ASCII!

   639  * some knowledge of the syntax itself.

   555  *

   640  *

   556  *       Since the IL language needs to know the location of newline

   641  * NOTE: Our definition of whitespace will only work in ASCII!

   557  *       (token EOL -> '\n' ), we need one definition of whitespace

   642  *

   558  *       for each language...

   559  */

   560 /*

   572 st_whitespace_only	[ \f\n\r\t\v]*

   654 st_whitespace			[ \f\n\r\t\v]*

   573 il_whitespace_only	[ \f\r\t\v]*

   655 il_whitespace			[ \f\r\t\v]*

   574 

   656 

   575 st_whitespace_text	{st_whitespace_only}|{comment}|{pragma}

   657 st_whitespace_or_pragma_or_commentX	({st_whitespace})|({pragma})|({comment})

   576 il_whitespace_text	{il_whitespace_only}|{comment}|{pragma}

   658 il_whitespace_or_pragma_or_commentX	({il_whitespace})|({pragma})|({comment})

   577 

   659 

   578 st_whitespace	{st_whitespace_text}*

   660 st_whitespace_or_pragma_or_comment	{st_whitespace_or_pragma_or_commentX}*

   579 il_whitespace	{il_whitespace_text}*

   661 il_whitespace_or_pragma_or_comment	{il_whitespace_or_pragma_or_commentX}*

   580 

   662 

   581 st_whitespace_text_no_pragma	{st_whitespace_only}|{comment}

   663 

   582 il_whitespace_text_no_pragma	{il_whitespace_only}|{comment}

   664 

   583 

   665 qualified_identifier	{identifier}(\.{identifier})+

   584 st_whitespace_no_pragma	{st_whitespace_text_no_pragma}*

   585 il_whitespace_no_pragma	{il_whitespace_text_no_pragma}*

   586 

   587 qualified_identifier	{identifier}(\.{identifier})*

   856 <body_state>{disable_code_generation_pragma}   return disable_code_generation_pragma_token;

   933 <body_state,vardecl_list_state>{disable_code_generation_pragma}   return disable_code_generation_pragma_token;

   857 <body_state>{enable_code_generation_pragma}    return enable_code_generation_pragma_token;

   934 <body_state,vardecl_list_state>{enable_code_generation_pragma}    return enable_code_generation_pragma_token;

   862          int cut = yytext[1]=='{'?2:1;

   939 		 int cut = yytext[1]=='{'?2:1;

   867 <body_state>{pragma} {/* return the pragmma without the enclosing '{' and '}' */

   944 <body_state,vardecl_list_state>{pragma} {/* return the pragmma without the enclosing '{' and '}' */

   869          yytext[strlen(yytext)-cut] = '\0';

   946 		 yytext[strlen(yytext)-cut] = '\0';

   953 	/* INITIAL -> decl_state */

  1030 	/* INITIAL -> header_state */

   969 	 *       continue in the <decl_state> state, untill the end of the FUNCTION, FUNCTION_BLOCK

  1046 	 *       continue in the <vardecl_state> state, untill the end of the FUNCTION, FUNCTION_BLOCK

   972 FUNCTION				BEGIN(decl_state); return FUNCTION;

  1049 FUNCTION				yy_push_state(header_state); return FUNCTION;

   973 FUNCTION_BLOCK				BEGIN(decl_state); return FUNCTION_BLOCK;

  1050 FUNCTION_BLOCK				yy_push_state(header_state); return FUNCTION_BLOCK;

   974 PROGRAM					BEGIN(decl_state); return PROGRAM;

  1051 PROGRAM					yy_push_state(header_state); return PROGRAM;

   992 	/* decl_state -> (body_state | sfc_state) */

  1069 	/* header_state -> (vardecl_list_state) */

   993 <decl_state>{

  1070 <header_state>{

   994 END_VAR{st_whitespace}VAR		{unput_text(strlen("END_VAR")); 

  1071 VAR				| /* execute the next rule's action, i.e. fall-through! */

   995 					 return END_VAR;

  1072 VAR_INPUT			|

   996 					}

  1073 VAR_OUTPUT			|

   997 END_VAR{st_whitespace}INITIAL_STEP	{unput_text(strlen("END_VAR")); 

  1074 VAR_IN_OUT			|

   998 					 yy_push_state(sfc_state); 

  1075 VAR_EXTERNAL			|

   999 					 return END_VAR;

  1076 VAR_GLOBAL			|

  1000 					}

  1077 VAR_TEMP			|

  1001 END_VAR{st_whitespace}			{unput_text(strlen("END_VAR")); 

  1078 VAR_CONFIG			|

  1002 					 cmd_goto_body_state(); 

  1079 VAR_ACCESS			unput_text(0); BEGIN(vardecl_list_state);

  1003 					 return END_VAR;

  1080 }

  1004 					}

  1081 

  1005 }

  1082 

  1006 

  1083 	/* vardecl_list_state -> (vardecl_state | body_state | INITIAL) */

  1007 	/* body_state -> (il_state | st_state) */

  1084 <vardecl_list_state>{

  1085 VAR_INPUT			| /* execute the next rule's action, i.e. fall-through! */

  1086 VAR_OUTPUT			|

  1087 VAR_IN_OUT			|

  1088 VAR_EXTERNAL			|

  1089 VAR_GLOBAL			|

  1090 VAR_TEMP			|

  1091 VAR_CONFIG			|

  1092 VAR_ACCESS			|

  1093 VAR				unput_text(0); yy_push_state(vardecl_state);

  1094 

  1095 END_FUNCTION			unput_text(0); BEGIN(INITIAL); 

  1096 END_FUNCTION_BLOCK		unput_text(0); BEGIN(INITIAL); 

  1097 END_PROGRAM			unput_text(0); BEGIN(INITIAL); 

  1098 

  1099 .				unput_text(0); yy_push_state(body_state); /* anything else, just change to body_state! */

  1100 }

  1101 

  1102 

  1103 	/* vardecl_list_state -> pop to $previous_state (vardecl_list_state) */

  1104 <vardecl_state>{

  1105 END_VAR				yy_pop_state(); return END_VAR; /* pop back to header_state */

  1106 }

  1107 

  1108 

  1109 	/* body_state -> (il_state | st_state | sfc_state) */

  1009 {st_whitespace_no_pragma}			/* Eat any whitespace */

  1111 INITIAL_STEP			unput_text(0); BEGIN(sfc_state); 

  1010 {qualified_identifier}{st_whitespace}":="	unput_text(0); BEGIN(st_state);

  1112 

  1011 {direct_variable_standard}{st_whitespace}":="	unput_text(0); BEGIN(st_state);

  1113 {qualified_identifier}		unput_text(0); BEGIN(st_state); /* will always be followed by '[' for an array access, or ':=' as the left hand of an assignment statement */

  1012 {qualified_identifier}"["			unput_text(0); BEGIN(st_state);

  1114 {direct_variable_standard}	unput_text(0); BEGIN(st_state); /* will always be followed by ':=' as the left hand of an assignment statement */

  1013 

  1115 

  1014 RETURN						unput_text(0); BEGIN(st_state);

  1116 RETURN				unput_text(0); BEGIN(st_state);

  1015 IF						unput_text(0); BEGIN(st_state);

  1117 IF				unput_text(0); BEGIN(st_state);

  1016 CASE						unput_text(0); BEGIN(st_state);

  1118 CASE				unput_text(0); BEGIN(st_state);

  1017 FOR						unput_text(0); BEGIN(st_state);

  1119 FOR				unput_text(0); BEGIN(st_state);

  1018 WHILE						unput_text(0); BEGIN(st_state);

  1120 WHILE				unput_text(0); BEGIN(st_state);

  1019 REPEAT						unput_text(0); BEGIN(st_state);

  1121 EXIT				unput_text(0); BEGIN(st_state);

  1020 EXIT						unput_text(0); BEGIN(st_state);

  1122 REPEAT				unput_text(0); BEGIN(st_state);

  1023 :=						unput_text(0); BEGIN(st_state);

  1125 :=				unput_text(0); BEGIN(st_state);

  1024 

  1025 	/* Hopefully, the above rules (along with the last one),

  1026          * used to distinguish ST from IL, are 

  1027 	 * enough to handle all ocurrences. However, if

  1028 	 * there is some situation where the compiler is getting confused,

  1029 	 * we add the following rule to detect 'label:' in IL code. This will

  1030 	 * allow the user to insert a label right at the beginning (which

  1031 	 * will probably not be used further by his code) simply as a way

  1032 	 * to force the compiler to interpret his code as IL code.

  1033 	 */

  1034 {identifier}{st_whitespace}":"{st_whitespace}	unput_text(0); BEGIN(il_state);

  1037 		 if (token == prev_declared_fb_name_token) {

  1128 		 if ((token == prev_declared_fb_name_token) || (token == prev_declared_variable_name_token)) {

  1038 		   /* the code has a call to a function block */

  1129 		   /* the code has a call to a function block OR has an assingment with a variable as the lvalue */

  1039 		   /* NOTE: if we ever decide to allow the user to use IL operator tokens

  1130 		   unput_text(0); BEGIN(st_state);

  1040 		    * (LD, ST, ...) as identifiers for variable names (including

  1131 		 } else

  1041 		    * function block instances), then the above inference/conclusion 

  1132  		 if (token == prev_declared_derived_function_name_token) {

  1042 		    * may be incorrect, and this condition may have to be changed!

  1133 		   /* the code has a call to a function - must be IL */

  1043 		    */	

  1134 		   unput_text(0); BEGIN(il_state);

  1044 		   BEGIN(st_state);

  1046 		   BEGIN(il_state);

  1136 		   /* Might be a lable in IL, or a bug in ST/IL code. We jump to IL */

  1137 		   unput_text(0); BEGIN(il_state);

  1051 .		unput_text(0); BEGIN(il_state);

  1141 .		unput_text(0); BEGIN(il_state); /* Don't know what it could be. This is most likely a bug. Let's just to a random state... */

  1054 	/* (il_state | st_state) -> $previous_state (decl_state or sfc_state) */

  1144 

  1145 

  1146 	/* (il_state | st_state) -> pop to $previous_state (vardecl_list_state or sfc_state) */

  1063 	/* sfc_state -> INITIAL */

  1155 	/* sfc_state -> pop to $previous_state (vardecl_list_state or sfc_state) */

  1086 <INITIAL,config_state,decl_state,st_state,sfc_state,task_init_state,sfc_qualifier_state>{st_whitespace_no_pragma}	/* Eat any whitespace */

  1172 	/* The whitespace */

  1087 <il_state>{il_whitespace_no_pragma}		/* Eat any whitespace */

  1173 <INITIAL,header_state,config_state,body_state,vardecl_list_state,vardecl_state,st_state,sfc_state,task_init_state,sfc_qualifier_state>{st_whitespace}	/* Eat any whitespace */

  1088 

  1174 <il_state>{il_whitespace}		/* Eat any whitespace */

  1089 

  1175 

  1176 	/* The comments */

  1177 <body_state,vardecl_list_state>{comment_beg}		yy_push_state(comment_state);

  1178 {comment_beg}						yy_push_state(comment_state);

  1179 <comment_state>{

  1180 {comment_beg}						{if (get_opt_nested_comments()) yy_push_state(comment_state);}

  1181 {comment_end}						yy_pop_state();

  1182 .							/* Ignore text inside comment! */

  1183 \n							/* Ignore text inside comment! */

  1184 }

  1642 <st_state>{identifier}/({st_whitespace})"=>"	{yylval.ID=strdup(yytext); return sendto_identifier_token;}

  1737 <st_state>{identifier}/({st_whitespace_or_pragma_or_comment})"=>"	{yylval.ID=strdup(yytext); return sendto_identifier_token;}

  1643 <il_state>{identifier}/({il_whitespace})"=>"	{yylval.ID=strdup(yytext); return sendto_identifier_token;}

  1738 <il_state>{identifier}/({il_whitespace_or_pragma_or_comment})"=>"	{yylval.ID=strdup(yytext); return sendto_identifier_token;}