1

/*

     2

 *  matiec - a compiler for the programming languages defined in IEC 61131-3

     3

 *

     4

 *  Copyright (C) 2003-2011  Mario de Sousa (msousa@fe.up.pt)

     5

 *

     6

 *  This program is free software: you can redistribute it and/or modify

     7

 *  it under the terms of the GNU General Public License as published by

     8

 *  the Free Software Foundation, either version 3 of thest_whitespaceLicense, or

     9

 *  (at your option) any later version.

    10

 *

    11

 *  This program is distributed in the hope that it will be useful,

    12

 *  but WITHOUT ANY WARRANTY; without even the implied warranty of

    13

 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    14

 *  GNU General Public License for more details.

    15

 *

    16

 *  You should have received a copy of the GNU General Public License

    17

 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.

    18

 *

    19

 *

    20

 * This code is made available on the understanding that it will not be

    21

 * used in safety-critical situations without a full and competent review.

    22

 */

    23

    24

/*

    25

 * An IEC 61131-3 compiler.

    26

 *

    27

 * Based on the

    28

 * FINAL DRAFT - IEC 61131-3, 2nd Ed. (2001-12-10)

    29

 *

    30

 */

    31

    32

/*

    33

 * Stage 1

    34

 * =======

    35

 *

    36

 * This file contains the lexical tokens definitions, from which

    37

 * the flex utility will generate a lexical parser function.

    38

 */

    39

    40

    41

    42

    43

/*****************************/

    44

/* Lexical Parser Options... */

    45

/*****************************/

    46

    47

/* The lexical analyser will never work in interactive mode,

    48

 * i.e., it will only process programs saved to files, and never

    49

 * programs being written inter-actively by the user.

    50

 * This option saves the resulting parser from calling the

    51

 * isatty() function, that seems to be generating some compile

    52

 * errors under some (older?) versions of flex.

    53

 */

    54

%option never-interactive

    55

    56

/* Have the lexical analyser use a 'char *yytext' instead of an

    57

 * array of char 'char yytext[??]' to store the lexical token.

    58

 */

    59

%pointer

    60

    61

    62

/* Have the lexical analyser ignore the case of letters.

    63

 * This will occur for all the tokens and keywords, but

    64

 * the resulting text handed up to the syntax parser

    65

 * will not be changed, and keep the original case

    66

 * of the letters in the input file.

    67

 */

    68

%option case-insensitive

    69

    70

/* Have the generated lexical analyser keep track of the

    71

 * line number it is currently analysing.

    72

 * This is used to pass up to the syntax parser

    73

 * the number of the line on which the current

    74

 * token was found. It will enable the syntax parser

    75

 * to generate more informatve error messages...

    76

 */

    77

%option yylineno

    78

    79

/* required for the use of the yy_pop_state() and

    80

 * yy_push_state() functions

    81

 */

    82

%option stack

    83

    84

/* The '%option stack' also requests the inclusion of 

    85

 * the yy_top_state(), however this function is not

    86

 * currently being used. This means that the compiler

    87

 * is complaining about the existance of this function.

    88

 * The following option removes the yy_top_state()

    89

 * function from the resulting c code, so the compiler 

    90

 * no longer complains.

    91

 */

    92

%option noyy_top_state

    93

    94

/* We will be using unput() in our flex code, so we cannot set the following option!... */

    95

/*

    96

%option nounput

    97

*/

    98

    99

/* The '%option debug' makes the generated scanner run in

   100

 * debug mode.

   101

%option debug

   102

 */

   103

   104

/**************************************************/

   105

/* External Variable and Function declarations... */

   106

/**************************************************/

   107

   108

   109

%{

   110

/* Define TEST_MAIN to include a main() function.

   111

 * Useful for testing the parser generated by flex.

   112

 */

   113

/*

   114

#define TEST_MAIN

   115

*/

   116

/* If lexical parser is compiled by itself, we need to define the following

   117

 * constant to some string. Under normal circumstances LIBDIRECTORY is set

   118

 * in the syntax parser header file...

   119

 */

   120

#ifdef TEST_MAIN

   121

#define DEFAULT_LIBDIR "just_testing"

   122

#endif

   123

   124

   125

   126

/* Required for strdup() */

   127

#include <string.h>

   128

   129

/* Required only for the declaration of abstract syntax classes

   130

 * (class symbol_c; class token_c; class list_c;)

   131

 * These will not be used in flex, but the token type union defined

   132

 * in iec_bison.hh contains pointers to these classes, so we must include

   133

 * it here.

   134

 */

   135

#include "../absyntax/absyntax.hh"

   136

   137

   138

/* iec_bison.hh is generated by bison.

   139

 * Contains the definition of the token constants, and the

   140

 * token value type YYSTYPE (in our case, a 'const char *')

   141

 */

   142

#include "iec_bison.hh"

   143

#include "stage1_2_priv.hh"

   144

   145

   146

/* Variable defined by the bison parser,

   147

 * where the value of the tokens will be stored

   148

 */

   149

extern YYSTYPE yylval;

   150

   151

/* The name of the file currently being parsed...

   152

 * Note that flex accesses and updates this global variable

   153

 * apropriately whenever it comes across an (*#include <filename> *) directive...

   154

 */

   155

const char *current_filename = NULL;

   156

   157

   158

   159

/* Variable defined by the bison parser.

   160

 * It must be initialised with the location

   161

 * of the token being parsed.

   162

 * This is only needed if we want to keep

   163

 * track of the locations, in order to give

   164

 * more meaningful error messages!

   165

 */

   166

/*

   167

 *extern YYLTYPE yylloc;

   168

b*/

   169

#define YY_INPUT(buf,result,max_size)  {\

   170

    result = GetNextChar(buf, max_size);\

   171

    if (  result <= 0  )\

   172

      result = YY_NULL;\

   173

    }

   174

   175

   176

/* Macro that is executed for every action.

   177

 * We use it to pass the location of the token

   178

 * back to the bison parser...

   179

 */

   180

#define YY_USER_ACTION {\

   181

	previous_tracking   =*current_tracking;					\

   182

	yylloc.first_line   = current_tracking->lineNumber;			\

   183

	yylloc.first_column = current_tracking->currentChar;			\

   184

	yylloc.first_file   = current_filename;					\

   185

	yylloc.first_order  = current_order;					\

   186

	\

   187

	UpdateTracking(yytext);							\

   188

	\

   189

	yylloc.last_line    = current_tracking->lineNumber;			\

   190

	yylloc.last_column  = current_tracking->currentChar - 1;		\

   191

	yylloc.last_file    = current_filename;					\

   192

	yylloc.last_order   = current_order;					\

   193

	\

   194

	current_tracking->currentTokenStart = current_tracking->currentChar;	\

   195

	current_order++;							\

   196

	}

   197

   198

   199

/* Since this lexical parser we defined only works in ASCII based

   200

 * systems, we might as well make sure it is being compiled on

   201

 * one...

   202

 * Lets check a few random characters...

   203

 */

   204

#if (('a' != 0x61) || ('A' != 0x41) || ('z' != 0x7A) || ('Z' != 0x5A) || \

   205

     ('0' != 0x30) || ('9' != 0x39) || ('(' != 0x28) || ('[' != 0x5B))

   206

#error This lexical analyser is not portable to a non ASCII based system.

   207

#endif

   208

   209

   210

/* Function only called from within flex, but defined

   211

 * in iec.y!

   212

 * We declare it here...

   213

 *

   214

 * Search for a symbol in either of the two symbol tables

   215

 * and return the token id of the first symbol found.

   216

 * Searches first in the variables, and only if not found

   217

 * does it continue searching in the library elements

   218

 */

   219

//token_id_t get_identifier_token(const char *identifier_str);

   220

int get_identifier_token(const char *identifier_str);

   221

%}

   222

   223

   224

/***************************************************/

   225

/* Forward Declaration of functions defined later. */

   226

/***************************************************/

   227

   228

%{

   229

void UpdateTracking(const char *text);

   230

/* return the character back to the input stream. */

   231

void unput_char(const char c);

   232

/* return all the text in the current token back to the input stream. */

   233

void unput_text(int n);

   234

/* return all the text in the current token back to the input stream, 

   235

 * but first return to the stream an additional character to mark the end of the token. 

   236

 */

   237

void unput_and_mark(const char mark_char);

   238

   239

void include_file(const char *include_filename);

   240

   241

/* The body_state tries to find a ';' before a END_PROGRAM, END_FUNCTION or END_FUNCTION_BLOCK or END_ACTION

   242

 * and ignores ';' inside comments and pragmas. This means that we cannot do this in a signle lex rule.

   243

 * Body_state therefore stores ALL text we consume in every rule, so we can push it back into the buffer

   244

 * once we have decided if we are parsing ST or IL code. The following functions manage that buffer used by

   245

 * the body_state.

   246

 */

   247

void  append_bodystate_buffer(const char *text, int is_whitespace = 0);

   248

void   unput_bodystate_buffer(void);

   249

int  isempty_bodystate_buffer(void);

   250

   251

int GetNextChar(char *b, int maxBuffer);

   252

%}

   253

   254

   255

   256

/****************************/

   257

/* Lexical Parser States... */

   258

/****************************/

   259

   260

/* NOTE: Our psrser can parse st or il code, intermixed

   261

 *       within the same file.

   262

 *       With IL we come across the issue of the EOL (end of line) token.

   263

 *       ST, and the declaration parts of IL do not use this token!

   264

 *       If the lexical analyser were to issue this token during ST

   265

 *       language parsing, or during the declaration of data types,

   266

 *       function headers, etc. in IL, the syntax parser would crash.

   267

 *

   268

 *       We can solve this issue using one of three methods:

   269

 *        (1) Augment all the syntax that does not accept the EOL

   270

 *            token to simply ignore it. This makes the syntax

   271

 *            definition (in iec.y) very cluttered!

   272

 *        (2) Let the lexical parser figure out which language

   273

 *            it is parsing, and decide whether or not to issue

   274

 *            the EOL token. This requires the lexical parser

   275

 *            to have knowledge of the syntax!, making for a poor

   276

 *            overall organisation of the code. It would also make it

   277

 *            very difficult to understand the lexical parser as it

   278

 *            would use several states, and a state machine to transition

   279

 *            between the states. The state transitions would be

   280

 *            intermingled with the lexical parser defintion!

   281

 *        (3) Use a mixture of (1) and (2). The lexical analyser

   282

 *            merely distinguishes between function headers and function

   283

 *            bodies, but no longer makes a distinction between il and

   284

 *            st language bodies. When parsing a body, it will return

   285

 *            the EOL token. In other states '\n' will be ignored as

   286

 *            whitespace.

   287

 *            The ST language syntax has been augmented in the syntax

   288

 *            parser configuration to ignore any EOL tokens that it may

   289

 *            come across!

   290

 *            This option has both drawbacks of option (1) and (2), but

   291

 *            much less intensely.

   292

 *            The syntax that gets cluttered is limited to the ST statements

   293

 *            (which is rather limited, compared to the function headers and

   294

 *            data type declarations, etc...), while the state machine in

   295

 *            the lexical parser becomes very simple. All state transitions

   296

 *            can be handled within the lexical parser by itself, and can be

   297

 *            easily identified. Thus knowledge of the syntax required by

   298

 *            the lexical parser is very limited!

   299

 *

   300

 * Amazingly enough, I (Mario) got to implement option (3)

   301

 * at first, requiring two basic states, decl and body.

   302

 * The lexical parser will enter the body state when

   303

 * it is parsing the body of a function/program/function block. The

   304

 * state transition is done when we find a VAR_END that is not followed

   305

 * by a VAR! This is the syntax knowledge that gets included in the

   306

 * lexical analyser with this option!

   307

 * Unfortunately, getting the st syntax parser to ignore EOL anywhere

   308

 * where they might appear leads to conflicts. This is due to the fact

   309

 * that the syntax parser uses the single look-ahead token to remove

   310

 * possible conflicts. When we insert a possible EOL, the single

   311

 * look ahead token becomes the EOL, which means the potential conflicts

   312

 * could no longer be resolved.

   313

 * Removing these conflicts would make the st syntax parser very convoluted,

   314

 * and adding the extraneous EOL would make it very cluttered.

   315

 * This option was therefore dropped in favour of another!

   316

 *

   317

 * I ended up implementing (2). Unfortunately the lexical analyser can

   318

 * not easily distinguish between il and st code, since function

   319

 * calls in il are very similar to function block calls in st.

   320

 * We therefore use an extra 'body' state. When the lexical parser

   321

 * finds that last END_VAR, it enters the body state. This state

   322

 * must figure out what language is being parsed from the first few

   323

 * tokens, and switch to the correct state (st, il or sfc) according to the

   324

 * language. This means that we insert quite a bit of knowledge of the

   325

 * syntax of the languages into the lexical parser. This is ugly, but it

   326

 * works, and at least it is possible to keep all the state changes together

   327

 * to make it easier to remove them later on if need be.

   328

 * Once the language being parsed has been identified, 

   329

 * the body state returns any matched text back to the buffer with unput(),

   330

 * to be later matched correctly by the apropriate language parser (st, il or sfc).

   331

 *

   332

 * Aditionally, in sfc state it may further recursively enter the body state

   333

 * once again. This is because an sfc body may contain ACTIONS, which are then

   334

 * written in one of the three languages (ST, IL or SFC), so once again we need

   335

 * to figure out which language the ACTION in the SFC was written in. We already

   336

 * ahve all that done in the body state, so we recursively transition to the body 

   337

 * state once again.

   338

 * Note that in this case, when coming out of the st/il state (whichever language

   339

 * the action was written in) the sfc state will become active again. This is done by

   340

 * pushing and poping the previously active state!

   341

 *

   342

 * The sfc_qualifier_state is required because when parsing actions within an

   343

 * sfc, we will be expecting action qualifiers (N, P, R, S, DS, SD, ...). In order

   344

 * to bison to work correctly, these qualifiers must be returned as tokens. However,

   345

 * these tokens are not reserved keywords, which means it should be possible to

   346

 * define variables/functions/FBs with any of these names (including 

   347

 * S and R which are special because they are also IL operators). So, when we are not

   348

 * expecting any action qualifiers, flex does not return these tokens, and is free

   349

 * to interpret them as previously defined variables/functions/... as the case may be.

   350

 *

   351

 * The time_literal_state is required because TIME# literals are decomposed into 

   352

 * portions, and wewant to send these portions one by one to bison. Each poertion will 

   353

 * represent the value in days/hours/minutes/seconds/ms.

   354

 * Unfortunately, some of these portions may also be lexically analysed as an identifier. So,

   355

 * we need to disable lexical identification of identifiers while parsing TIME# literals!

   356

 * e.g.:  TIME#55d_4h_56m

   357

 *       We would like to return to bison the tokens 'TIME' '#' '55d' '_' '4h' '_' '56m'

   358

 *       Unfortunately, flex will join '_' and '4h' to create a legal {identifier} '_4h',

   359

 *       and return that identifier instead! So, we added this state!

   360

 *

   361

 * The ignore_pou_state state is only used when bison says it is doing the pre-parsing.

   362

 * During pre-parsing, the main state machine will only transition between

   363

 * INITIAL and ignore_pou_state, and from here back to INITIAL. All other

   364

 * transitions are inhibited. This inhibition is actually just enforced by making

   365

 * sure that the INITIAL ---> ignore_pou_state transition is tested before all other

   366

 * transitions coming out of INITIAL state. All other transitions are unaffected, as they

   367

 * never get a chance to be evaluated when bison is doing pre-parsing.

   368

 * Pre-parsing is a first quick scan through the whole input source code simply

   369

 * to determine the list of POUs and datatypes that will be defined in that

   370

 * code. Basically, the objective is to fill up the previously_declared_xxxxx

   371

 * maps, without processing the code itself. Once these maps have been filled up,

   372

 * bison will throw away the AST (abstract syntax tree) created up to that point, 

   373

 * and scan through the same source code again, but this time creating a correct AST.

   374

 * This pre-scan allows the source code to reference POUs and datatypes that are

   375

 * only declared after they are used!

   376

 * 

   377

 *

   378

 * Here is a main state machine...

   379

 *                                                                         --+  

   380

 *                                                                           |  these states are

   381

 *              +------------> get_pou_name_state  ----> ignore_pou_state    |  only active 

   382

 *              |                                            |               |  when bison is 

   383

 *              |  ------------------------------------------+               |  doing the 

   384

 *              |  |                                                         |  pre-parsing!!

   385

 *              |  v                                                       --+

   386

 *       +---> INITIAL <-------> config

   387

 *       |        \

   388

 *       |        V

   389

 *       |   header_state

   390

 *       |        |

   391

 *       |        V

   392

 *     vardecl_list_state <------> var_decl

   393

 *       ^        | 

   394

 *       |        | [using push()]

   395

 *       |        |

   396

 *       |        V

   397

 *       |       body, 

   398

 *       |        |

   399

 *       |        | 

   400

 *       |   -------------------

   401

 *       |   |       |         |

   402

 *       |   v       v         v

   403

 *       |  st      il        sfc

   404

 *       |   |       |         |  [using pop() when leaving st/il/sfc => goes to vardecl_list_state]

   405

 *       |   |       |         |

   406

 *       -----------------------

   407

 *

   408

 * NOTE:- When inside sfc, and an action or transition in ST/IL is found, then 

   409

 *        we also push() to the body state. This means that sometimes, when pop()ing

   410

 *        from st and il, the state machine may return to the sfc state!

   411

 *      - The transitions form sfc to body will be decided by bison, which will

   412

 *        tell flex to do the transition by calling cmd_goto_body_state().

   413

 *   

   414

 * 

   415

 * Possible state changes are:

   416

 *   INITIAL -> goto(ignore_pou_state)

   417

 *               (This transition state is only used when bison says it is doing the pre-parsing.)

   418

 *               (This transition takes precedence over all other transitions!)

   419

 *               (when a FUNCTION, FUNCTION_BLOCK, PROGRAM or CONFIGURATION is found)

   420

 * 

   421

 *   INITIAL -> goto(config_state)

   422

 *                (when a CONFIGURATION is found)

   423

 * 

   424

 *   INITIAL -> goto(header_state)

   425

 *               (when a FUNCTION, FUNCTION_BLOCK, or PROGRAM is found)

   426

 * 

   427

 *   header_state -> goto(vardecl_list_state)

   428

 *               (When the first VAR token is found, i.e. at begining of first VAR .. END_VAR declaration)

   429

 * 

   430

 *  vardecl_list_state -> push current state (vardecl_list_state), and goto(vardecl_state) 

   431

 *                (when a VAR token is found)

   432

 *   vardecl_state -> pop() to (vardecl_list_state) 

   433

 *                (when a END_VAR token is found)

   434

 * 

   435

 *   vardecl_list_state -> push current state (vardecl_list_state), and goto(body_state) 

   436

 *                (when the last END_VAR is found!)

   437

 *

   438

 *   body_state    -> goto(sfc_state)

   439

 *                     (when it figures out it is parsing sfc language)

   440

 *   body_state    -> goto(st_state)

   441

 *                     (when it figures out it is parsing st language)

   442

 *   body_state    -> goto(il_state)

   443

 *                     (when it figures out it is parsing il language)

   444

 *   st_state      -> pop() to vardecl_list_state

   445

 *                     (when a END_FUNCTION, END_FUNCTION_BLOCK, END_PROGRAM,

   446

 *                      END_ACTION or END_TRANSITION is found)

   447

 *   il_state      -> pop() to vardecl_list_state

   448

 *                     (when a END_FUNCTION, END_FUNCTION_BLOCK, END_PROGRAM,

   449

 *                      END_ACTION or END_TRANSITION is found)

   450

 *   sfc_state     -> pop() to vardecl_list_state

   451

 *                     (when a END_FUNCTION, END_FUNCTION_BLOCK, or END_PROGRAM is found)

   452

 * 

   453

 *   ignore_pou_state   -> goto(INITIAL)

   454

 *                         (when a END_FUNCTION, END_FUNCTION_BLOCK, END_PROGRAM or END_CONFIGURATION is found)

   455

 *   vardecl_list_state -> goto(INITIAL)

   456

 *                         (when a END_FUNCTION, END_FUNCTION_BLOCK, or END_PROGRAM is found)

   457

 *   config_state       -> goto(INITIAL)

   458

 *                         (when a END_CONFIGURATION is found)

   459

 * 

   460

 *  

   461

 *   sfc_state     -> push current state(sfc_state); goto(body_state)

   462

 *                     (when parsing an action. This transition is requested by bison)

   463

 *   sfc_state     -> push current state(sfc_state); goto(sfc_qualifier_state)

   464

 *                     (when expecting an action qualifier. This transition is requested by bison)

   465

 *   sfc_qualifier_state -> pop() to sfc_state

   466

 *                     (when no longer expecting an action qualifier. This transition is requested by bison)

   467

 *

   468

 *   config_state  -> push(config_state); goto(task_init_state)

   469

 *                     (when parsing a task initialisation. This transition is requested by bison)

   470

 *   task_init_state -> pop()

   471

 *                     (when no longer parsing task initialisation parameters. This transition is requested by bison)

   472

 *

   473

 * 

   474

 * There is another secondary state machine for parsing comments, another for file_includes, 

   475

 * and yet another for time literals.

   476

 */

   477

   478

   479

/* Bison is in the pre-parsing stage, and we are parsing a POU. Ignore everything up to the end of the POU! */

   480

%x ignore_pou_state

   481

%x get_pou_name_state

   482

   483

/* we are parsing a configuration. */

   484

%s config_state

   485

   486

/* Inside a configuration, we are parsing a task initialisation parameters */

   487

/* This means that PRIORITY, SINGLE and INTERVAL must be handled as

   488

 * tokens, and not as possible identifiers. Note that the above words

   489

 * are not keywords.

   490

 */

   491

%s task_init_state

   492

   493

/* we are looking for the first VAR inside a function's, program's or function block's declaration */

   494

/* This is not exclusive (%x) as we must be able to parse the identifier and data types of a function/FB */

   495

%s header_state

   496

   497

/* we are parsing a function, program or function block sequence of VAR..END_VAR delcarations */

   498

%x vardecl_list_state 

   499

/* a substate of the vardecl_list_state: we are inside a specific VAR .. END_VAR */

   500

%s vardecl_state

   501

   502

/* we will be parsing a function body/action/transition. Whether il/st/sfc remains to be determined */

   503

%x body_state

   504

   505

/* we are parsing il code -> flex must return the EOL tokens!       */