1

/*

     2

 * (c) 2003 Mario de Sousa

     3

 *

     4

 * Offered to the public under the terms of the GNU General Public License

     5

 * as published by the Free Software Foundation; either version 2 of the

     6

 * License, or (at your option) any later version.

     7

 *

     8

 * This program is distributed in the hope that it will be useful, but

     9

 * WITHOUT ANY WARRANTY; without even the implied warranty of

    10

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General

    11

 * Public License for more details.

    12

 *

    13

 * This code is made available on the understanding that it will not be

    14

 * used in safety-critical situations without a full and competent review.

    15

 */

    16

    17

/*

    18

 * An IEC 61131-3 IL and ST compiler.

    19

 *

    20

 * Based on the

    21

 * FINAL DRAFT - IEC 61131-3, 2nd Ed. (2001-12-10)

    22

 *

    23

 */

    24

    25

/*

    26

 * Stage 1

    27

 * =======

    28

 *

    29

 * This file contains the lexical tokens definitions, from which

    30

 * the flex utility will generate a lexical parser function.

    31

 */

    32

    33

    34

    35

    36

/*****************************/

    37

/* Lexical Parser Options... */

    38

/*****************************/

    39

    40

/* The lexical analyser will never work in interactive mode,

    41

 * i.e., it will only process programs saved to files, and never

    42

 * programs being written inter-actively by the user.

    43

 * This option saves the resulting parser from calling the

    44

 * isatty() function, that seems to be generating some compile

    45

 * errors under some (older?) versions of flex.

    46

 */

    47

%option never-interactive

    48

    49

/* Have the lexical analyser use a 'char *yytext' instead of an

    50

 * array of char 'char yytext[??]' to store the lexical token.

    51

 */

    52

%pointer

    53

    54

    55

/* Have the lexical analyser ignore the case of letters.

    56

 * This will occur for all the tokens and keywords, but

    57

 * the resulting text handed up to the syntax parser

    58

 * will not be changed, and keep the original case

    59

 * of the letters in the input file.

    60

 */

    61

%option case-insensitive

    62

    63

/* Have the generated lexical analyser keep track of the

    64

 * line number it is currently analysing.

    65

 * This is used to pass up to the syntax parser

    66

 * the number of the line on which the current

    67

 * token was found. It will enable the syntax parser

    68

 * to generate more informatve error messages...

    69

 */

    70

%option yylineno

    71

    72

/* required for the use of the yy_pop_state() and

    73

 * yy_push_state() functions

    74

 */

    75

%option stack

    76

    77

/* The '%option stack' also requests the inclusion of 

    78

 * the yy_top_state(), however this function is not

    79

 * currently being used. This means that the compiler

    80

 * is complaining about the existance of this function.

    81

 * The following option removes the yy_top_state()

    82

 * function from the resulting c code, so the compiler 

    83

 * no longer complains.

    84

 */

    85

%option noyy_top_state

    86

    87

/**************************************************/

    88

/* External Variable and Function declarations... */

    89

/**************************************************/

    90

    91

    92

%{

    93

/* Define TEST_MAIN to include a main() function.

    94

 * Useful for testing the parser generated by flex.

    95

 */

    96

/*

    97

#define TEST_MAIN

    98

*/

    99

/* If lexical parser is compiled by itself, we need to define the following

   100

 * constant to some string. Under normal circumstances LIBDIRECTORY is set

   101

 * in the syntax parser header file...

   102

 */

   103

#ifdef TEST_MAIN

   104

#define LIBDIRECTORY "just_testing"

   105

#endif

   106

   107

   108

   109

/* Required for strdup() */

   110

#include <string.h>

   111

   112

/* Required only for the declaration of abstract syntax classes

   113

 * (class symbol_c; class token_c; class list_c;)

   114

 * These will not be used in flex, but the token type union defined

   115

 * in iec.hh contains pointers to these classes, so we must include

   116

 * it here.

   117

 */

   118

#include "../absyntax/absyntax.hh"

   119

   120

   121

/* generated by bison.

   122

 * Contains the definition of the token constants, and the

   123

 * token value type YYSTYPE (in our case, a 'const char *')

   124

 */

   125

#include "iec.y.hh"

   126

#include "stage1_2_priv.hh"

   127

   128

   129

/* Variable defined by the bison parser,

   130

 * where the value of the tokens will be stored

   131

 */

   132

extern YYSTYPE yylval;

   133

   134

/* The name of the file currently being parsed...

   135

 * This variable is declared and read from the code generated by bison!

   136

 * Note that flex accesses and updates this global variable

   137

 * apropriately whenever it comes across an (*#include <filename> *)

   138

 * directive...

   139

 */

   140

/*

   141

 NOTE: already defined in iec.y.hh 

   142

extern const char *current_filename;

   143

*/

   144

   145

   146

/* We will not be using unput() in our flex code... */

   147

#define YY_NO_UNPUT

   148

   149

/* Variable defined by the bison parser.

   150

 * It must be initialised with the location

   151

 * of the token being parsed.

   152

 * This is only needed if we want to keep

   153

 * track of the locations, in order to give

   154

 * more meaningful error messages!

   155

 */

   156

extern YYLTYPE yylloc;

   157

   158

/* Macro that is executed for every action.

   159

 * We use it to pass the location of the token

   160

 * back to the bison parser...

   161

 */

   162

#define YY_USER_ACTION { 					\

   163

	yylloc.first_line = yylloc.last_line = yylineno;	\

   164

	yylloc.first_column = yylloc.last_column = 0;		\

   165

	}

   166

   167

   168

/* Since this lexical parser we defined only works in ASCII based

   169

 * systems, we might as well make sure it is being compiled on

   170

 * one...

   171

 * Lets check a few random characters...

   172

 */

   173

#if (('a' != 0x61) || ('A' != 0x41) || ('z' != 0x7A) || ('Z' != 0x5A) || \

   174

     ('0' != 0x30) || ('9' != 0x39) || ('(' != 0x28) || ('[' != 0x5B))

   175

#error This lexical analyser is not portable to a non ASCII based system.

   176

#endif

   177

   178

   179

/* Function only called from within flex, but defined

   180

 * in iec.y!

   181

 * We declare it here...

   182

 *

   183

 * Search for a symbol in either of the two symbol tables

   184

 * and return the token id of the first symbol found.

   185

 * Searches first in the variables, and only if not found

   186

 * does it continue searching in the library elements

   187

 */

   188

//token_id_t get_identifier_token(const char *identifier_str);

   189

int get_identifier_token(const char *identifier_str);

   190

%}

   191

   192

   193

/***************************************************/

   194

/* Forward Declaration of functions defined later. */

   195

/***************************************************/

   196

   197

%{

   198

/* return all the text in the current token back to the input stream. */

   199

void unput_text(unsigned int n);

   200

%}

   201

   202

   203

   204

/****************************/

   205

/* Lexical Parser States... */

   206

/****************************/

   207

   208

/* NOTE: Our psrser can parse st or il code, intermixed

   209

 *       within the same file.

   210

 *       With IL we come across the issue of the EOL (end of line) token.

   211

 *       ST, and the declaration parts of IL do not use this token!

   212

 *       If the lexical analyser were to issue this token during ST

   213

 *       language parsing, or during the declaration of data types,

   214

 *       function headers, etc. in IL, the syntax parser would crash.

   215

 *

   216

 *       We can solve this issue using one of three methods:

   217

 *        (1) Augment all the syntax that does not accept the EOL

   218

 *            token to simply ignore it. This makes the syntax

   219

 *            definition (in iec.y) very cluttered!

   220

 *        (2) Let the lexical parser figure out which language

   221

 *            it is parsing, and decide whether or not to issue

   222

 *            the EOL token. This requires the lexical parser

   223

 *            to have knowledge of the syntax!, making for a poor

   224

 *            overall organisation of the code. It would also make it

   225

 *            very difficult to understand the lexical parser as it

   226

 *            would use several states, and a state machine to transition

   227

 *            between the states. The state transitions would be

   228

 *            intermingled with the lexical parser defintion!

   229

 *        (3) Use a mixture of (1) and (2). The lexical analyser

   230

 *            merely distinguishes between function headers and function

   231

 *            bodies, but no longer makes a distinction between il and

   232

 *            st language bodies. When parsing a body, it will return

   233

 *            the EOL token. In other states '\n' will be ignored as

   234

 *            whitespace.

   235

 *            The ST language syntax has been augmented in the syntax

   236

 *            parser configuration to ignore any EOL tokens that it may

   237

 *            come across!

   238

 *            This option has both drawbacks of option (1) and (2), but

   239

 *            much less intensely.

   240

 *            The syntax that gets cluttered is limited to the ST statements

   241

 *            (which is rather limited, compared to the function headers and

   242

 *            data type declarations, etc...), while the state machine in

   243

 *            the lexical parser becomes very simple. All state transitions

   244

 *            can be handled within the lexical parser by itself, and can be

   245

 *            easily identified. Thus knowledge of the syntax required by

   246

 *            the lexical parser is very limited!

   247

 *

   248

 * Amazingly enough, I (Mario) got to implement option (3)

   249

 * at first, requiring two basic states, decl and body.

   250

 * The lexical parser will enter the body state when

   251

 * it is parsing the body of a function/program/function block. The

   252

 * state transition is done when we find a VAR_END that is not followed

   253

 * by a VAR! This is the syntax knowledge that gets included in the

   254

 * lexical analyser with this option!

   255

 * Unfortunately, getting the st syntax parser to ignore EOL anywhere

   256

 * where they might appear leads to conflicts. This is due to the fact

   257

 * that the syntax parser uses the single look-ahead token to remove

   258

 * possible conflicts. When we insert a possible EOL, the single

   259

 * look ahead token becomes the EOL, which means the potential conflicts

   260

 * could no longer be resolved.

   261

 * Removing these conflicts would make the st syntax parser very convoluted,

   262

 * and adding the extraneous EOL would make it very cluttered.

   263

 * This option was therefore dropped in favour of another!

   264

 *

   265

 * I ended up implementing (2). Unfortunately the lexical analyser can

   266

 * not easily distinguish between il and st code, since function

   267

 * calls in il are very similar to function block calls in st.

   268

 * We therefore use an extra 'body' state. When the lexical parser

   269

 * finds that last END_VAR, it enters the body state. This state

   270

 * must figure out what language is being parsed from the first few

   271

 * tokens, and switch to the correct state (st or il) according to the

   272

 * language. This means that we insert quite a bit of knowledge of the

   273

 * syntax of the languages into the lexical parser. This is ugly, but it

   274

 * works, and at least it is possible to keep all the state changes together

   275

 * to make it easier to remove them later on if need be.

   276

 * The body state returns any matched text back to the buffer with unput(),

   277

 * to be later matched correctly by the apropriate language parser (st or il).

   278

 * The state machine has 6 possible states (INITIAL, config, decl, body, st, il)

   279

 * Possible state changes are:

   280

 *   INITIAL -> decl_state (when a FUNCTION, FUNCTION_BLOCK, or PROGRAM is found,

   281

 *                    and followed by a VAR declaration)

   282

 *   INITIAL -> il_st_state (when a FUNCTION, FUNCTION_BLOCK, or PROGRAM is found,

   283

 *                    and _not_ followed by a VAR declaration)

   284

 *   INITIAL -> config_state (when a CONFIGURATION is found)

   285

 *   decl_state    -> il_st_state (when the last END_VAR is found, i.e. the function body starts)

   286

 *   il_st_state   -> sfc_state (when it figures out it is parsing sfc language)

   287

 *   il_st_state   -> st_state (when it figures out it is parsing st language)

   288

 *   il_st_state   -> il_state (when it figures out it is parsing il language)

   289

 *   decl_state    -> INITIAL (when a END_FUNCTION, END_FUNCTION_BLOCK, or END_PROGRAM is found)

   290

 *   st_state      -> INITIAL (when a END_FUNCTION, END_FUNCTION_BLOCK, or END_PROGRAM is found)

   291

 *   sfc_state     -> INITIAL (when a END_FUNCTION, END_FUNCTION_BLOCK, or END_PROGRAM is found)

   292

 *   il_state      -> INITIAL (when a END_FUNCTION, END_FUNCTION_BLOCK, or END_PROGRAM is found)

   293

 *   config_state  -> INITIAL (when a END_CONFIGURATION is found)

   294

 */

   295

/* we are parsing a configuration. */

   296

%s config_state

   297

   298

/* we are parsing a function, program or function block declaration */

   299

%s decl_state

   300

   301

/* we will be parsing a function body. Whether il/st is remains unknown */

   302

%x il_st_state

   303

   304

/* we are parsing il code -> flex must return the EOL tokens!       */

   305

%s il_state

   306

   307

/* we are parsing st code -> flex must not return the EOL tokens!   */

   308

%s st_state

   309

   310

/* we are parsing sfc code -> flex must not return the EOL tokens!   */

   311

%s sfc_state

   312

   313

   314

   315

/*******************/

   316

/* File #include's */

   317

/*******************/

   318

   319

/* We extend the IEC 61131-3 standard syntax to allow inclusion

   320

 * of other files, using the IEC 61131-3 pragma directive...

   321

 * The accepted syntax is:

   322

 *  {#include "<filename>"}

   323

 */

   324

   325

/* the "include" states are used for picking up the name of an include file */

   326

%x include_beg

   327

%x include_filename

   328

%x include_end

   329

   330

   331

file_include_pragma_filename	[^\"]*

   332

file_include_pragma_beg		"{#include"{st_whitespace_only}\"

   333

file_include_pragma_end		\"{st_whitespace_only}"}"

   334

file_include_pragma			{file_include_pragma_beg}{file_include_pragma_filename}{file_include_pragma_end}

   335

   336

   337

%{

   338

#define MAX_INCLUDE_DEPTH 16

   339

   340

typedef struct {

   341

	  YY_BUFFER_STATE buffer_state;

   342

	  int lineno;

   343

	  const char *filename;

   344

	} include_stack_t;

   345

   346

include_stack_t include_stack[MAX_INCLUDE_DEPTH];

   347

int include_stack_ptr = 0;

   348

   349

const char *INCLUDE_DIRECTORIES[] = {

   350

	"",

   351

	"lib/",

   352

	"/lib/",

   353

	"/usr/lib/",

   354

	"/usr/lib/iec/",

   355

	LIBDIRECTORY "/",

   356

	NULL /* must end with NULL!! */

   357

	};

   358

   359

%}

   360

   361

   362

   363

/*****************************/

   364

/* Prelimenary constructs... */

   365

/*****************************/

   366

   367

   368

/* A pragma... */

   369

   370

pragma "{"[^}]*"}"

   371

   372

/* NOTE: this seemingly unnecessary complex definition is required

   373

 *       to be able to eat up comments such as:

   374

 *          '(* Testing... ! ***** ******)'

   375

 *       without using the trailing context command in flex (/{context})

   376

 *       since {comment} itself will later be used with

   377

 *       trailing context ({comment}/{context})

   378

 */

   379

not_asterisk				[^*]

   380

not_close_parenthesis_nor_asterisk	[^*)]

   381

asterisk				"*"

   382

comment_text		{not_asterisk}|(({asterisk}+){not_close_parenthesis_nor_asterisk})

   383

   384

comment		"(*"({comment_text}*)({asterisk}+)")"

   385

   386

   387

/*

   388

3.1 Whitespace

   389

 (NOTE: Whitespace IS clearly defined, to include newline!!! See section 2.1.4!!!)

   390

 No definition of whitespace is given, in other words, the characters that may be used to seperate language tokens are not pecisely defined. One may nevertheless make an inteligent guess of using the space (' '), and other characters also commonly considered whitespace in other programming languages (horizontal tab, vertical tab, form feed, etc.).

   391

 The main question is whether the newline character should be considered whitespace. IL language statements use an EOL token (End Of Line) to distinguish between some language constructs. The EOL token itself is openly defined as "normally consist[ing] of the 'paragraph separator' ", leaving the final choice open to each implemention. If we choose the newline character to represent the EOL token, it may then not be considered whitespace.

   392

 On the other hand, some examples that come in a non-normative annex of the specification allow function declarations to span multiple3.1 Whitespace

   393

 (NOTE: Whitespace IS clearly defined, to include newline!!! See section 2.1.4!!!)

   394

 No definition of whitespace is given, in other words, the characters that may be used to seperate language tokens are not pecisely defined. One may nevertheless make an inteligent guess of using the space (' '), and other characters also commonly considered whitespace in other programming languages (horizontal tab, vertical tab, form feed, etc.).

   395

 The main question is whether the newline character should be considered whitespace. IL language statements use an EOL token (End Of Line) to distinguish between some language constructs. The EOL token itself is openly defined as "normally consist[ing] of the 'paragraph separator' ", leaving the final choice open to each implemention. If we choose the newline character to represent the EOL token, it may then not be considered whitespace.

   396

 On the other hand, some examples that come in a non-normative annex of the specification allow function declarations to span multiple lines, which means that the newline character is being considered as whitespace.

   397

 Our implementation works around this issue by including the new line character in the whitespace while parsing function declarations and the ST language, and parsing it as the EOL token only while parsing IL language statements. This requires the use of a state machine in the lexical parser that needs at least some knowledge of the syntax itself.

   398

*/

   399

/* NOTE: Our definition of whitespace will only work in ASCII!

   400

 *

   401

 *       Since the IL language needs to know the location of newline

   402

 *       (token EOL -> '\n' ), we need one definition of whitespace

   403

 *       for each language...

   404

 */

   405

/*

   406

 * NOTE: we cannot use

   407

 *         st_whitespace	[:space:]*

   408

 *       since we use {st_whitespace} as trailing context. In our case

   409

 *       this would not constitute "dangerous trailing context", but the

   410

 *       lexical generator (i.e. flex) does not know this (since it does

   411

 *       not know which characters belong to the set [:space:]), and will

   412

 *       generate a "dangerous trailing context" warning!

   413

 *       We use this alternative just to stop the flex utility from

   414

 *       generating the invalid (in this case) warning...

   415

 */

   416

   417

st_whitespace_only	[ \f\n\r\t\v]*

   418

il_whitespace_only	[ \f\r\t\v]*

   419

   420

st_whitespace_text	{st_whitespace_only}|{comment}|{pragma}

   421

il_whitespace_text	{il_whitespace_only}|{comment}|{pragma}

   422

   423

st_whitespace	{st_whitespace_text}*

   424

il_whitespace	{il_whitespace_text}*

   425

   426

st_whitespace_text_no_pragma	{st_whitespace_only}|{comment}

   427

il_whitespace_text_no_pragma	{il_whitespace_only}|{comment}

   428

   429

st_whitespace_no_pragma	{st_whitespace_text_no_pragma}*

   430

il_whitespace_no_pragma	{il_whitespace_text_no_pragma}*

   431

   432

qualified_identifier	{identifier}(\.{identifier})?

   433

   434

   435

   436

/*****************************************/

   437

/* B.1.1 Letters, digits and identifiers */

   438

/*****************************************/

   439

/* NOTE: The following definitions only work if the host computer

   440

 *       is using the ASCII maping. For e.g., with EBCDIC [A-Z]

   441

 *       contains non-alphabetic characters!

   442

 *       The correct way of doing it would be to use

   443

 *       the [:upper:] etc... definitions.

   444

 *

   445

 *       Unfortunately, further on we need all printable

   446

 *       characters (i.e. [:print:]), but excluding '$'.

   447

 *       Flex does not allow sets to be composed by excluding

   448

 *       elements. Sets may only be constructed by adding new

   449

 *       elements, which means that we have to revert to

   450

 *       [\x20\x21\x23\x25\x26\x28-x7E] for the definition

   451

 *       of the printable characters with the required exceptions.

   452

 *       The above also implies the use of ASCII, but now we have

   453

 *       no way to work around it|

   454

 *

   455

 *       The conclusion is that our parser is limited to ASCII

   456

 *       based host computers!!

   457

 */

   458

letter		[A-Za-z]

   459

digit		[0-9]

   460

octal_digit	[0-7]

   461

hex_digit	{digit}|[A-F]

   462

identifier	({letter}|(_({letter}|{digit})))((_?({letter}|{digit}))*)

   463

   464

   465

/*******************/

   466

/* B.1.2 Constants */

   467

/*******************/

   468

   469

/******************************/

   470

/* B.1.2.1   Numeric literals */

   471

/******************************/

   472

integer         {digit}((_?{digit})*)

   473

binary_integer  2#{bit}((_?{bit})*)

   474

bit		[0-1]

   475

octal_integer   8#{octal_digit}((_?{octal_digit})*)

   476

hex_integer     16#{hex_digit}((_?{hex_digit})*)

   477

exponent        [Ee]([+-]?){integer}

   478

/* The correct definition for real would be:

   479

 * real		{integer}\.{integer}({exponent}?)

   480

 *

   481

 * Unfortunately, the spec also defines fixed_point (B 1.2.3.1) as:

   482

 * fixed_point		{integer}\.{integer}

   483

 *

   484

 * This means that {integer}\.{integer} could be interpreted

   485

 * as either a fixed_point or a real.

   486

 * I have opted to interpret {integer}\.{integer} as a fixed_point.

   487

 * In order to do this, the definition of real has been changed to:

   488

 * real		{integer}\.{integer}{exponent}

   489

 *

   490

 * This means that the syntax parser now needs to define a real to be

   491

 * either a real_token or a fixed_point_token!

   492

 */

   493

real		{integer}\.{integer}{exponent}

   494

   495

   496

/*******************************/

   497

/* B.1.2.2   Character Strings */

   498

/*******************************/

   499

/*

   500

common_character_representation :=

   501

<any printable character except '$', '"' or "'">

   502

|'$$'

   503

|'$L'|'$N'|'$P'|'$R'|'$T'

   504

|'$l'|'$n'|'$p'|'$r'|'$t'

   505

   506

NOTE: 	$ = 0x24

   507

	" = 0x22

   508

	' = 0x27

   509

   510

	printable chars in ASCII: 0x20-0x7E

   511

*/

   512

   513

esc_char_u		$L|$N|$P|$R|$T

   514

esc_char_l		$l|$n|$p|$r|$t

   515

esc_char		$$|{esc_char_u}|{esc_char_l}

   516

double_byte_char	(${hex_digit}{hex_digit}{hex_digit}{hex_digit})

   517

single_byte_char	(${hex_digit}{hex_digit})

   518

   519

/* WARNING:

   520

 * This definition is only valid in ASCII...

   521

 *

   522

 * Flex includes the function print_char() that defines

   523

 * all printable characters portably (i.e. whatever character

   524

 * encoding is currently being used , ASCII, EBCDIC, etc...)

   525

 * Unfortunately, we cannot generate the definition of

   526

 * common_character_representation portably, since flex