Lexical Analyzer

LEX (Lex/Yacc , flex/bison)

Ahmad Yoosofan

Compiler course

University of Kashan

https://web.cs.hacettepe.edu.tr/~bbm301/lectures/lecture3_lex.pdf

Install and Compile Lex File

1 %%
2 [0-9]+                 {printf("INT:%s\n",yytext);}
3 [a-zA-Z_][a-zA-Z_0-9]* {printf("ID :%s\n",yytext);}
4 .   ;
5 \n  ;
6 %%
7 
8 int main(){ 
9   yylex();
10   printf("\n");
11   return 0;
12 }
 1 # Install flex in Ubuntu
 2 
 3 sudo apt install g++
 4 
 5 sudo apt install flex
 6 
 7 # Compile Lex File
 8 
 9 flex 01.count_words.l
10 
11 gcc lex.yy.c -lfl
12 
13 ./a.out < count_words_input_1.txt
1 #line 3 "lex.yy.c"
2 
3 #define  YY_INT_ALIGNED short int
4 
5 /* A lexical scanner generated by flex */
6 
7 #define FLEX_SCANNER
8 #define YY_FLEX_MAJOR_VERSION 2
9 #define YY_FLEX_MINOR_VERSION 6
10 #define YY_FLEX_SUBMINOR_VERSION 4
11 #if YY_FLEX_SUBMINOR_VERSION > 0
12 #define FLEX_BETA
13 #endif
14 
15 /* First, we deal with  platform-specific or compiler-specific issues. */
16 
17 /* begin standard C headers. */
18 #include <stdio.h>
19 #include <string.h>
20 #include <errno.h>
21 #include <stdlib.h>
22 
23 /* end standard C headers. */
24 
25 /* flex integer type definitions */
26 
27 #ifndef FLEXINT_H
28 #define FLEXINT_H
29 
30 /* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */
31 
32 #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
33 
34 /* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
35  * if you want the limit (max/min) macros for int types. 
36  */
37 #ifndef __STDC_LIMIT_MACROS
38 #define __STDC_LIMIT_MACROS 1
39 #endif
40 
41 #include <inttypes.h>
42 typedef int8_t flex_int8_t;
43 typedef uint8_t flex_uint8_t;
44 typedef int16_t flex_int16_t;
45 typedef uint16_t flex_uint16_t;
46 typedef int32_t flex_int32_t;
47 typedef uint32_t flex_uint32_t;
48 #else
49 typedef signed char flex_int8_t;
50 typedef short int flex_int16_t;
51 typedef int flex_int32_t;
52 typedef unsigned char flex_uint8_t; 
53 typedef unsigned short int flex_uint16_t;
54 typedef unsigned int flex_uint32_t;
55 
56 /* Limits of integral types. */
57 #ifndef INT8_MIN
58 #define INT8_MIN               (-128)
59 #endif
60 #ifndef INT16_MIN
61 #define INT16_MIN              (-32767-1)
62 #endif
63 #ifndef INT32_MIN
64 #define INT32_MIN              (-2147483647-1)
65 #endif
66 #ifndef INT8_MAX
67 #define INT8_MAX               (127)
68 #endif
69 #ifndef INT16_MAX
70 #define INT16_MAX              (32767)
71 #endif
72 #ifndef INT32_MAX
73 #define INT32_MAX              (2147483647)
74 #endif
75 #ifndef UINT8_MAX
76 #define UINT8_MAX              (255U)
77 #endif
78 #ifndef UINT16_MAX
79 #define UINT16_MAX             (65535U)
80 #endif
81 #ifndef UINT32_MAX
82 #define UINT32_MAX             (4294967295U)
83 #endif
84 
85 #ifndef SIZE_MAX
86 #define SIZE_MAX               (~(size_t)0)
87 #endif
88 
89 #endif /* ! C99 */
90 
91 #endif /* ! FLEXINT_H */
92 
93 /* begin standard C++ headers. */
94 
95 /* TODO: this is always defined, so inline it */
96 #define yyconst const
97 
98 #if defined(__GNUC__) && __GNUC__ >= 3
99 #define yynoreturn __attribute__((__noreturn__))
100 #else
101 #define yynoreturn
102 #endif
103 
104 /* Returned upon end-of-file. */
105 #define YY_NULL 0
106 
107 /* Promotes a possibly negative, possibly signed char to an
108  *   integer in range [0..255] for use as an array index.
109  */
110 #define YY_SC_TO_UI(c) ((YY_CHAR) (c))
111 
112 /* Enter a start condition.  This macro really ought to take a parameter,
113  * but we do it the disgusting crufty way forced on us by the ()-less
114  * definition of BEGIN.
115  */
116 #define BEGIN (yy_start) = 1 + 2 *
117 /* Translate the current start state into a value that can be later handed
118  * to BEGIN to return to the state.  The YYSTATE alias is for lex
119  * compatibility.
120  */
121 #define YY_START (((yy_start) - 1) / 2)
122 #define YYSTATE YY_START
123 /* Action number for EOF rule of a given start state. */
124 #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1)
125 /* Special action meaning "start processing a new file". */
126 #define YY_NEW_FILE yyrestart( yyin  )
127 #define YY_END_OF_BUFFER_CHAR 0
128 
129 /* Size of default input buffer. */
130 #ifndef YY_BUF_SIZE
131 #ifdef __ia64__
132 /* On IA-64, the buffer size is 16k, not 8k.
133  * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case.
134  * Ditto for the __ia64__ case accordingly.
135  */
136 #define YY_BUF_SIZE 32768
137 #else
138 #define YY_BUF_SIZE 16384
139 #endif /* __ia64__ */
140 #endif
141 
142 /* The state buf must be large enough to hold one state per character in the main buffer.
143  */
144 #define YY_STATE_BUF_SIZE   ((YY_BUF_SIZE + 2) * sizeof(yy_state_type))
145 
146 #ifndef YY_TYPEDEF_YY_BUFFER_STATE
147 #define YY_TYPEDEF_YY_BUFFER_STATE
148 typedef struct yy_buffer_state *YY_BUFFER_STATE;
149 #endif
150 
151 #ifndef YY_TYPEDEF_YY_SIZE_T
152 #define YY_TYPEDEF_YY_SIZE_T
153 typedef size_t yy_size_t;
154 #endif
155 
156 extern int yyleng;
157 
158 extern FILE *yyin, *yyout;
159 
160 #define EOB_ACT_CONTINUE_SCAN 0
161 #define EOB_ACT_END_OF_FILE 1
162 #define EOB_ACT_LAST_MATCH 2
163     
164     #define YY_LESS_LINENO(n)
165     #define YY_LINENO_REWIND_TO(ptr)
166     
167 /* Return all but the first "n" matched characters back to the input stream. */
168 #define yyless(n) \
169         do \
170                 { \
171                 /* Undo effects of setting up yytext. */ \
172         int yyless_macro_arg = (n); \
173         YY_LESS_LINENO(yyless_macro_arg);\
174                 *yy_cp = (yy_hold_char); \
175                 YY_RESTORE_YY_MORE_OFFSET \
176                 (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \
177                 YY_DO_BEFORE_ACTION; /* set up yytext again */ \
178                 } \
179         while ( 0 )
180 #define unput(c) yyunput( c, (yytext_ptr)  )
181 
182 #ifndef YY_STRUCT_YY_BUFFER_STATE
183 #define YY_STRUCT_YY_BUFFER_STATE
184 struct yy_buffer_state
185         {
186         FILE *yy_input_file;
187 
188         char *yy_ch_buf;                /* input buffer */
189         char *yy_buf_pos;               /* current position in input buffer */
190 
191         /* Size of input buffer in bytes, not including room for EOB
192          * characters.
193          */
194         int yy_buf_size;
195 
196         /* Number of characters read into yy_ch_buf, not including EOB
197          * characters.
198          */
199         int yy_n_chars;
200 
201         /* Whether we "own" the buffer - i.e., we know we created it,
202          * and can realloc() it to grow it, and should free() it to
203          * delete it.
204          */
205         int yy_is_our_buffer;
206 
207         /* Whether this is an "interactive" input source; if so, and
208          * if we're using stdio for input, then we want to use getc()
209          * instead of fread(), to make sure we stop fetching input after
210          * each newline.
211          */
212         int yy_is_interactive;
213 
214         /* Whether we're considered to be at the beginning of a line.
215          * If so, '^' rules will be active on the next match, otherwise
216          * not.
217          */
218         int yy_at_bol;
219 
220     int yy_bs_lineno; /**< The line count. */
221     int yy_bs_column; /**< The column count. */
222 
223         /* Whether to try to fill the input buffer when we reach the
224          * end of it.
225          */
226         int yy_fill_buffer;
227 
228         int yy_buffer_status;
229 
230 #define YY_BUFFER_NEW 0
231 #define YY_BUFFER_NORMAL 1
232         /* When an EOF's been seen but there's still some text to process
233          * then we mark the buffer as YY_EOF_PENDING, to indicate that we
234          * shouldn't try reading from the input source any more.  We might
235          * still have a bunch of tokens to match, though, because of
236          * possible backing-up.
237          *
238          * When we actually see the EOF, we change the status to "new"
239          * (via yyrestart()), so that the user can continue scanning by
240          * just pointing yyin at a new input file.
241          */
242 #define YY_BUFFER_EOF_PENDING 2
243 
244         };
245 #endif /* !YY_STRUCT_YY_BUFFER_STATE */
246 
247 /* Stack of input buffers. */
248 static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */
249 static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */
250 static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */
251 
252 /* We provide macros for accessing buffer states in case in the
253  * future we want to put the buffer states in a more general
254  * "scanner state".
255  *
256  * Returns the top of the stack, or NULL.
257  */
258 #define YY_CURRENT_BUFFER ( (yy_buffer_stack) \
259                           ? (yy_buffer_stack)[(yy_buffer_stack_top)] \
260                           : NULL)
261 /* Same as previous macro, but useful when we know that the buffer stack is not
262  * NULL or when we need an lvalue. For internal use only.
263  */
264 #define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)]
265 
266 /* yy_hold_char holds the character lost when yytext is formed. */
267 static char yy_hold_char;
268 static int yy_n_chars;          /* number of characters read into yy_ch_buf */
269 int yyleng;
270 
271 /* Points to current character in buffer. */
272 static char *yy_c_buf_p = NULL;
273 static int yy_init = 0;         /* whether we need to initialize */
274 static int yy_start = 0;        /* start state number */
275 
276 /* Flag which is used to allow yywrap()'s to do buffer switches
277  * instead of setting up a fresh yyin.  A bit of a hack ...
278  */
279 static int yy_did_buffer_switch_on_eof;
280 
281 void yyrestart ( FILE *input_file  );
282 void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer  );
283 YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size  );
284 void yy_delete_buffer ( YY_BUFFER_STATE b  );
285 void yy_flush_buffer ( YY_BUFFER_STATE b  );
286 void yypush_buffer_state ( YY_BUFFER_STATE new_buffer  );
287 void yypop_buffer_state ( void );
288 
289 static void yyensure_buffer_stack ( void );
290 static void yy_load_buffer_state ( void );
291 static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file  );
292 #define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER )
293 
294 YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size  );
295 YY_BUFFER_STATE yy_scan_string ( const char *yy_str  );
296 YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len  );
297 
298 void *yyalloc ( yy_size_t  );
299 void *yyrealloc ( void *, yy_size_t  );
300 void yyfree ( void *  );
301 
302 #define yy_new_buffer yy_create_buffer
303 #define yy_set_interactive(is_interactive) \
304         { \
305         if ( ! YY_CURRENT_BUFFER ){ \
306         yyensure_buffer_stack (); \
307                 YY_CURRENT_BUFFER_LVALUE =    \
308             yy_create_buffer( yyin, YY_BUF_SIZE ); \
309         } \
310         YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \
311         }
312 #define yy_set_bol(at_bol) \
313         { \
314         if ( ! YY_CURRENT_BUFFER ){\
315         yyensure_buffer_stack (); \
316                 YY_CURRENT_BUFFER_LVALUE =    \
317             yy_create_buffer( yyin, YY_BUF_SIZE ); \
318         } \
319         YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \
320         }
321 #define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol)
322 
323 /* Begin user sect3 */
324 typedef flex_uint8_t YY_CHAR;
325 
326 FILE *yyin = NULL, *yyout = NULL;
327 
328 typedef int yy_state_type;
329 
330 extern int yylineno;
331 int yylineno = 1;
332 
333 extern char *yytext;
334 #ifdef yytext_ptr
335 #undef yytext_ptr
336 #endif
337 #define yytext_ptr yytext
338 
339 static yy_state_type yy_get_previous_state ( void );
340 static yy_state_type yy_try_NUL_trans ( yy_state_type current_state  );
341 static int yy_get_next_buffer ( void );
342 static void yynoreturn yy_fatal_error ( const char* msg  );
343 
344 /* Done after the current pattern has been matched and before the
345  * corresponding action - sets up yytext.
346  */
347 #define YY_DO_BEFORE_ACTION \
348         (yytext_ptr) = yy_bp; \
349         yyleng = (int) (yy_cp - yy_bp); \
350         (yy_hold_char) = *yy_cp; \
351         *yy_cp = '\0'; \
352         (yy_c_buf_p) = yy_cp;
353 #define YY_NUM_RULES 5
354 #define YY_END_OF_BUFFER 6
355 /* This struct is not used in this scanner,
356    but its presence is necessary. */
357 struct yy_trans_info
358         {
359         flex_int32_t yy_verify;
360         flex_int32_t yy_nxt;
361         };
362 static const flex_int16_t yy_accept[11] =
363     {   0,
364         0,    0,    6,    3,    4,    1,    2,    1,    2,    0
365     } ;
366 
367 static const YY_CHAR yy_ec[256] =
368     {   0,
369         1,    1,    1,    1,    1,    1,    1,    1,    1,    2,
370         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
371         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
372         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
373         1,    1,    1,    1,    1,    1,    1,    3,    3,    3,
374         3,    3,    3,    3,    3,    3,    3,    1,    1,    1,
375         1,    1,    1,    1,    4,    4,    4,    4,    4,    4,
376         4,    4,    4,    4,    4,    4,    4,    4,    4,    4,
377         4,    4,    4,    4,    4,    4,    4,    4,    4,    4,
378         1,    1,    1,    1,    4,    1,    4,    4,    4,    4,
379 
380         4,    4,    4,    4,    4,    4,    4,    4,    4,    4,
381         4,    4,    4,    4,    4,    4,    4,    4,    4,    4,
382         4,    4,    1,    1,    1,    1,    1,    1,    1,    1,
383         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
384         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
385         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
386         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
387         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
388         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
389         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
390 
391         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
392         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
393         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
394         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
395         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
396         1,    1,    1,    1,    1
397     } ;
398 
399 static const YY_CHAR yy_meta[5] =
400     {   0,
401         1,    1,    2,    3
402     } ;
403 
404 static const flex_int16_t yy_base[13] =
405     {   0,
406         0,    0,    8,    9,    9,    0,    0,    0,    0,    9,
407         5,    3
408     } ;
409 
410 static const flex_int16_t yy_def[13] =
411     {   0,
412        10,    1,   10,   10,   10,   11,   12,   11,   12,    0,
413        10,   10
414     } ;
415 
416 static const flex_int16_t yy_nxt[14] =
417     {   0,
418         4,    5,    6,    7,    9,    9,    8,   10,    3,   10,
419        10,   10,   10
420     } ;
421 
422 static const flex_int16_t yy_chk[14] =
423     {   0,
424         1,    1,    1,    1,   12,   12,   11,    3,   10,   10,
425        10,   10,   10
426     } ;
427 
428 static yy_state_type yy_last_accepting_state;
429 static char *yy_last_accepting_cpos;
430 
431 extern int yy_flex_debug;
432 int yy_flex_debug = 0;
433 
434 /* The intent behind this definition is that it'll catch
435  * any uses of REJECT which flex missed.
436  */
437 #define REJECT reject_used_but_not_detected
438 #define yymore() yymore_used_but_not_detected
439 #define YY_MORE_ADJ 0
440 #define YY_RESTORE_YY_MORE_OFFSET
441 char *yytext;
442 #line 1 "01.count_words.l"
443 #line 445 "lex.yy.c"
444 
445 #define INITIAL 0
446 
447 #ifndef YY_NO_UNISTD_H
448 /* Special case for "unistd.h", since it is non-ANSI. We include it way
449  * down here because we want the user's section 1 to have been scanned first.
450  * The user has a chance to override it with an option.
451  */
452 #include <unistd.h>
453 #endif
454 
455 #ifndef YY_EXTRA_TYPE
456 #define YY_EXTRA_TYPE void *
457 #endif
458 
459 static int yy_init_globals ( void );
460 
461 /* Accessor methods to globals.
462    These are made visible to non-reentrant scanners for convenience. */
463 
464 int yylex_destroy ( void );
465 
466 int yyget_debug ( void );
467 
468 void yyset_debug ( int debug_flag  );
469 
470 YY_EXTRA_TYPE yyget_extra ( void );
471 
472 void yyset_extra ( YY_EXTRA_TYPE user_defined  );
473 
474 FILE *yyget_in ( void );
475 
476 void yyset_in  ( FILE * _in_str  );
477 
478 FILE *yyget_out ( void );
479 
480 void yyset_out  ( FILE * _out_str  );
481 
482                         int yyget_leng ( void );
483 
484 char *yyget_text ( void );
485 
486 int yyget_lineno ( void );
487 
488 void yyset_lineno ( int _line_number  );
489 
490 /* Macros after this point can all be overridden by user definitions in
491  * section 1.
492  */
493 
494 #ifndef YY_SKIP_YYWRAP
495 #ifdef __cplusplus
496 extern "C" int yywrap ( void );
497 #else
498 extern int yywrap ( void );
499 #endif
500 #endif
501 
502 #ifndef YY_NO_UNPUT
503     
504     static void yyunput ( int c, char *buf_ptr  );
505     
506 #endif
507 
508 #ifndef yytext_ptr
509 static void yy_flex_strncpy ( char *, const char *, int );
510 #endif
511 
512 #ifdef YY_NEED_STRLEN
513 static int yy_flex_strlen ( const char * );
514 #endif
515 
516 #ifndef YY_NO_INPUT
517 #ifdef __cplusplus
518 static int yyinput ( void );
519 #else
520 static int input ( void );
521 #endif
522 
523 #endif
524 
525 /* Amount of stuff to slurp up with each read. */
526 #ifndef YY_READ_BUF_SIZE
527 #ifdef __ia64__
528 /* On IA-64, the buffer size is 16k, not 8k */
529 #define YY_READ_BUF_SIZE 16384
530 #else
531 #define YY_READ_BUF_SIZE 8192
532 #endif /* __ia64__ */
533 #endif
534 
535 /* Copy whatever the last rule matched to the standard output. */
536 #ifndef ECHO
537 /* This used to be an fputs(), but since the string might contain NUL's,
538  * we now use fwrite().
539  */
540 #define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0)
541 #endif
542 
543 /* Gets input and stuffs it into "buf".  number of characters read, or YY_NULL,
544  * is returned in "result".
545  */
546 #ifndef YY_INPUT
547 #define YY_INPUT(buf,result,max_size) \
548         if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
549                 { \
550                 int c = '*'; \
551                 int n; \
552                 for ( n = 0; n < max_size && \
553                              (c = getc( yyin )) != EOF && c != '\n'; ++n ) \
554                         buf[n] = (char) c; \
555                 if ( c == '\n' ) \
556                         buf[n++] = (char) c; \
557                 if ( c == EOF && ferror( yyin ) ) \
558                         YY_FATAL_ERROR( "input in flex scanner failed" ); \
559                 result = n; \
560                 } \
561         else \
562                 { \
563                 errno=0; \
564                 while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \
565                         { \
566                         if( errno != EINTR) \
567                                 { \
568                                 YY_FATAL_ERROR( "input in flex scanner failed" ); \
569                                 break; \
570                                 } \
571                         errno=0; \
572                         clearerr(yyin); \
573                         } \
574                 }\
575 \
576 
577 #endif
578 
579 /* No semi-colon after return; correct usage is to write "yyterminate();" -
580  * we don't want an extra ';' after the "return" because that will cause
581  * some compilers to complain about unreachable statements.
582  */
583 #ifndef yyterminate
584 #define yyterminate() return YY_NULL
585 #endif
586 
587 /* Number of entries by which start-condition stack grows. */
588 #ifndef YY_START_STACK_INCR
589 #define YY_START_STACK_INCR 25
590 #endif
591 
592 /* Report a fatal error. */
593 #ifndef YY_FATAL_ERROR
594 #define YY_FATAL_ERROR(msg) yy_fatal_error( msg )
595 #endif
596 
597 /* end tables serialization structures and prototypes */
598 
599 /* Default declaration of generated scanner - a define so the user can
600  * easily add parameters.
601  */
602 #ifndef YY_DECL
603 #define YY_DECL_IS_OURS 1
604 
605 extern int yylex (void);
606 
607 #define YY_DECL int yylex (void)
608 #endif /* !YY_DECL */
609 
610 /* Code executed at the beginning of each rule, after yytext and yyleng
611  * have been set up.
612  */
613 #ifndef YY_USER_ACTION
614 #define YY_USER_ACTION
615 #endif
616 
617 /* Code executed at the end of each rule. */
618 #ifndef YY_BREAK
619 #define YY_BREAK /*LINTED*/break;
620 #endif
621 
622 #define YY_RULE_SETUP \
623         YY_USER_ACTION
624 
625 /** The main scanner function which does all the work.
626  */
627 YY_DECL
628 {
629         yy_state_type yy_current_state;
630         char *yy_cp, *yy_bp;
631         int yy_act;
632     
633         if ( !(yy_init) )
634                 {
635                 (yy_init) = 1;
636 
637 #ifdef YY_USER_INIT
638                 YY_USER_INIT;
639 #endif
640 
641                 if ( ! (yy_start) )
642                         (yy_start) = 1; /* first start state */
643 
644                 if ( ! yyin )
645                         yyin = stdin;
646 
647                 if ( ! yyout )
648                         yyout = stdout;
649 
650                 if ( ! YY_CURRENT_BUFFER ) {
651                         yyensure_buffer_stack ();
652                         YY_CURRENT_BUFFER_LVALUE =
653                                 yy_create_buffer( yyin, YY_BUF_SIZE );
654                 }
655 
656                 yy_load_buffer_state(  );
657                 }
658 
659         {
660 #line 1 "01.count_words.l"
661 
662 #line 664 "lex.yy.c"
663 
664         while ( /*CONSTCOND*/1 )                /* loops until end-of-file is reached */
665                 {
666                 yy_cp = (yy_c_buf_p);
667 
668                 /* Support of yytext. */
669                 *yy_cp = (yy_hold_char);
670 
671                 /* yy_bp points to the position in yy_ch_buf of the start of
672                  * the current run.
673                  */
674                 yy_bp = yy_cp;
675 
676                 yy_current_state = (yy_start);
677 yy_match:
678                 do
679                         {
680                         YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)] ;
681                         if ( yy_accept[yy_current_state] )
682                                 {
683                                 (yy_last_accepting_state) = yy_current_state;
684                                 (yy_last_accepting_cpos) = yy_cp;
685                                 }
686                         while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
687                                 {
688                                 yy_current_state = (int) yy_def[yy_current_state];
689                                 if ( yy_current_state >= 11 )
690                                         yy_c = yy_meta[yy_c];
691                                 }
692                         yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
693                         ++yy_cp;
694                         }
695                 while ( yy_base[yy_current_state] != 9 );
696 
697 yy_find_action:
698                 yy_act = yy_accept[yy_current_state];
699                 if ( yy_act == 0 )
700                         { /* have to back up */
701                         yy_cp = (yy_last_accepting_cpos);
702                         yy_current_state = (yy_last_accepting_state);
703                         yy_act = yy_accept[yy_current_state];
704                         }
705 
706                 YY_DO_BEFORE_ACTION;
707 
708 do_action:      /* This label is used only to access EOF actions. */
709 
710                 switch ( yy_act )
711         { /* beginning of action switch */
712                         case 0: /* must back up */
713                         /* undo the effects of YY_DO_BEFORE_ACTION */
714                         *yy_cp = (yy_hold_char);
715                         yy_cp = (yy_last_accepting_cpos);
716                         yy_current_state = (yy_last_accepting_state);
717                         goto yy_find_action;
718 
719 case 1:
720 YY_RULE_SETUP
721 #line 2 "01.count_words.l"
722 {printf("INT:%s\n",yytext);}
723         YY_BREAK
724 case 2:
725 YY_RULE_SETUP
726 #line 3 "01.count_words.l"
727 {printf("ID :%s\n",yytext);}
728         YY_BREAK
729 case 3:
730 YY_RULE_SETUP
731 #line 4 "01.count_words.l"
732 ;
733         YY_BREAK
734 case 4:
735 /* rule 4 can match eol */
736 YY_RULE_SETUP
737 #line 5 "01.count_words.l"
738 ;
739         YY_BREAK
740 case 5:
741 YY_RULE_SETUP
742 #line 6 "01.count_words.l"
743 ECHO;
744         YY_BREAK
745 #line 747 "lex.yy.c"
746 case YY_STATE_EOF(INITIAL):
747         yyterminate();
748 
749         case YY_END_OF_BUFFER:
750                 {
751                 /* Amount of text matched not including the EOB char. */
752                 int yy_amount_of_matched_text = (int) (yy_cp - (yytext_ptr)) - 1;
753 
754                 /* Undo the effects of YY_DO_BEFORE_ACTION. */
755                 *yy_cp = (yy_hold_char);
756                 YY_RESTORE_YY_MORE_OFFSET
757 
758                 if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW )
759                         {
760                         /* We're scanning a new file or input source.  It's
761                          * possible that this happened because the user
762                          * just pointed yyin at a new source and called
763                          * yylex().  If so, then we have to assure
764                          * consistency between YY_CURRENT_BUFFER and our
765                          * globals.  Here is the right place to do so, because
766                          * this is the first action (other than possibly a
767                          * back-up) that will match for the new input source.
768                          */
769                         (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars;
770                         YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin;
771                         YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL;
772                         }
773 
774                 /* Note that here we test for yy_c_buf_p "<=" to the position
775                  * of the first EOB in the buffer, since yy_c_buf_p will
776                  * already have been incremented past the NUL character
777                  * (since all states make transitions on EOB to the
778                  * end-of-buffer state).  Contrast this with the test
779                  * in input().
780                  */
781                 if ( (yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] )
782                         { /* This was really a NUL. */
783                         yy_state_type yy_next_state;
784 
785                         (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text;
786 
787                         yy_current_state = yy_get_previous_state(  );
788 
789                         /* Okay, we're now positioned to make the NUL
790                          * transition.  We couldn't have
791                          * yy_get_previous_state() go ahead and do it
792                          * for us because it doesn't know how to deal
793                          * with the possibility of jamming (and we don't
794                          * want to build jamming into it because then it
795                          * will run more slowly).
796                          */
797 
798                         yy_next_state = yy_try_NUL_trans( yy_current_state );
799 
800                         yy_bp = (yytext_ptr) + YY_MORE_ADJ;
801 
802                         if ( yy_next_state )
803                                 {
804                                 /* Consume the NUL. */
805                                 yy_cp = ++(yy_c_buf_p);
806                                 yy_current_state = yy_next_state;
807                                 goto yy_match;
808                                 }
809 
810                         else
811                                 {
812                                 yy_cp = (yy_c_buf_p);
813                                 goto yy_find_action;
814                                 }
815                         }
816 
817                 else switch ( yy_get_next_buffer(  ) )
818                         {
819                         case EOB_ACT_END_OF_FILE:
820                                 {
821                                 (yy_did_buffer_switch_on_eof) = 0;
822 
823                                 if ( yywrap(  ) )
824                                         {
825                                         /* Note: because we've taken care in
826                                          * yy_get_next_buffer() to have set up
827                                          * yytext, we can now set up
828                                          * yy_c_buf_p so that if some total
829                                          * hoser (like flex itself) wants to
830                                          * call the scanner after we return the
831                                          * YY_NULL, it'll still work - another
832                                          * YY_NULL will get returned.
833                                          */
834                                         (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ;
835 
836                                         yy_act = YY_STATE_EOF(YY_START);
837                                         goto do_action;
838                                         }
839 
840                                 else
841                                         {
842                                         if ( ! (yy_did_buffer_switch_on_eof) )
843                                                 YY_NEW_FILE;
844                                         }
845                                 break;
846                                 }
847 
848                         case EOB_ACT_CONTINUE_SCAN:
849                                 (yy_c_buf_p) =
850                                         (yytext_ptr) + yy_amount_of_matched_text;
851 
852                                 yy_current_state = yy_get_previous_state(  );
853 
854                                 yy_cp = (yy_c_buf_p);
855                                 yy_bp = (yytext_ptr) + YY_MORE_ADJ;
856                                 goto yy_match;
857 
858                         case EOB_ACT_LAST_MATCH:
859                                 (yy_c_buf_p) =
860                                 &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)];
861 
862                                 yy_current_state = yy_get_previous_state(  );
863 
864                                 yy_cp = (yy_c_buf_p);
865                                 yy_bp = (yytext_ptr) + YY_MORE_ADJ;
866                                 goto yy_find_action;
867                         }
868                 break;
869                 }
870 
871         default:
872                 YY_FATAL_ERROR(
873                         "fatal flex scanner internal error--no action found" );
874         } /* end of action switch */
875                 } /* end of scanning one token */
876         } /* end of user's declarations */
877 } /* end of yylex */
878 
879 /* yy_get_next_buffer - try to read in a new buffer
880  *
881  * Returns a code representing an action:
882  *      EOB_ACT_LAST_MATCH -
883  *      EOB_ACT_CONTINUE_SCAN - continue scanning from current position
884  *      EOB_ACT_END_OF_FILE - end of file
885  */
886 static int yy_get_next_buffer (void)
887 {
888         char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf;
889         char *source = (yytext_ptr);
890         int number_to_move, i;
891         int ret_val;
892 
893         if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] )
894                 YY_FATAL_ERROR(
895                 "fatal flex scanner internal error--end of buffer missed" );
896 
897         if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 )
898                 { /* Don't try to fill the buffer, so this is an EOF. */
899                 if ( (yy_c_buf_p) - (yytext_ptr) - YY_MORE_ADJ == 1 )
900                         {
901                         /* We matched a single character, the EOB, so
902                          * treat this as a final EOF.
903                          */
904                         return EOB_ACT_END_OF_FILE;
905                         }
906 
907                 else
908                         {
909                         /* We matched some text prior to the EOB, first
910                          * process it.
911                          */
912                         return EOB_ACT_LAST_MATCH;
913                         }
914                 }
915 
916         /* Try to read more data. */
917 
918         /* First move last chars to start of buffer. */
919         number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr) - 1);
920 
921         for ( i = 0; i < number_to_move; ++i )
922                 *(dest++) = *(source++);
923 
924         if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING )
925                 /* don't do the read, it's not guaranteed to return an EOF,
926                  * just force an EOF
927                  */
928                 YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0;
929 
930         else
931                 {
932                         int num_to_read =
933                         YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1;
934 
935                 while ( num_to_read <= 0 )
936                         { /* Not enough room in the buffer - grow it. */
937 
938                         /* just a shorter name for the current buffer */
939                         YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE;
940 
941                         int yy_c_buf_p_offset =
942                                 (int) ((yy_c_buf_p) - b->yy_ch_buf);
943 
944                         if ( b->yy_is_our_buffer )
945                                 {
946                                 int new_size = b->yy_buf_size * 2;
947 
948                                 if ( new_size <= 0 )
949                                         b->yy_buf_size += b->yy_buf_size / 8;
950                                 else
951                                         b->yy_buf_size *= 2;
952 
953                                 b->yy_ch_buf = (char *)
954                                         /* Include room in for 2 EOB chars. */
955                                         yyrealloc( (void *) b->yy_ch_buf,
956                                                          (yy_size_t) (b->yy_buf_size + 2)  );
957                                 }
958                         else
959                                 /* Can't grow it, we don't own it. */
960                                 b->yy_ch_buf = NULL;
961 
962                         if ( ! b->yy_ch_buf )
963                                 YY_FATAL_ERROR(
964                                 "fatal error - scanner input buffer overflow" );
965 
966                         (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset];
967 
968                         num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size -
969                                                 number_to_move - 1;
970 
971                         }
972 
973                 if ( num_to_read > YY_READ_BUF_SIZE )
974                         num_to_read = YY_READ_BUF_SIZE;
975 
976                 /* Read in more data. */
977                 YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]),
978                         (yy_n_chars), num_to_read );
979 
980                 YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars);
981                 }
982 
983         if ( (yy_n_chars) == 0 )
984                 {
985                 if ( number_to_move == YY_MORE_ADJ )
986                         {
987                         ret_val = EOB_ACT_END_OF_FILE;
988                         yyrestart( yyin  );
989                         }
990 
991                 else
992                         {
993                         ret_val = EOB_ACT_LAST_MATCH;
994                         YY_CURRENT_BUFFER_LVALUE->yy_buffer_status =
995                                 YY_BUFFER_EOF_PENDING;
996                         }
997                 }
998 
999         else
1000                 ret_val = EOB_ACT_CONTINUE_SCAN;
1001 
1002         if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) {
1003                 /* Extend the array by 50%, plus the number we really need. */
1004                 int new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1);
1005                 YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc(
1006                         (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size  );
1007                 if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
1008                         YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" );
1009                 /* "- 2" to take care of EOB's */
1010                 YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2);
1011         }
1012 
1013         (yy_n_chars) += number_to_move;
1014         YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR;
1015         YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR;
1016 
1017         (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0];
1018 
1019         return ret_val;
1020 }
1021 
1022 /* yy_get_previous_state - get the state just before the EOB char was reached */
1023 
1024     static yy_state_type yy_get_previous_state (void)
1025 {
1026         yy_state_type yy_current_state;
1027         char *yy_cp;
1028     
1029         yy_current_state = (yy_start);
1030 
1031         for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp )
1032                 {
1033                 YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1);
1034                 if ( yy_accept[yy_current_state] )
1035                         {
1036                         (yy_last_accepting_state) = yy_current_state;
1037                         (yy_last_accepting_cpos) = yy_cp;
1038                         }
1039                 while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
1040                         {
1041                         yy_current_state = (int) yy_def[yy_current_state];
1042                         if ( yy_current_state >= 11 )
1043                                 yy_c = yy_meta[yy_c];
1044                         }
1045                 yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
1046                 }
1047 
1048         return yy_current_state;
1049 }
1050 
1051 /* yy_try_NUL_trans - try to make a transition on the NUL character
1052  *
1053  * synopsis
1054  *      next_state = yy_try_NUL_trans( current_state );
1055  */
1056     static yy_state_type yy_try_NUL_trans  (yy_state_type yy_current_state )
1057 {
1058         int yy_is_jam;
1059         char *yy_cp = (yy_c_buf_p);
1060 
1061         YY_CHAR yy_c = 1;
1062         if ( yy_accept[yy_current_state] )
1063                 {
1064                 (yy_last_accepting_state) = yy_current_state;
1065                 (yy_last_accepting_cpos) = yy_cp;
1066                 }
1067         while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
1068                 {
1069                 yy_current_state = (int) yy_def[yy_current_state];
1070                 if ( yy_current_state >= 11 )
1071                         yy_c = yy_meta[yy_c];
1072                 }
1073         yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
1074         yy_is_jam = (yy_current_state == 10);
1075 
1076                 return yy_is_jam ? 0 : yy_current_state;
1077 }
1078 
1079 #ifndef YY_NO_UNPUT
1080 
1081     static void yyunput (int c, char * yy_bp )
1082 {
1083         char *yy_cp;
1084     
1085     yy_cp = (yy_c_buf_p);
1086 
1087         /* undo effects of setting up yytext */
1088         *yy_cp = (yy_hold_char);
1089 
1090         if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 )
1091                 { /* need to shift things up to make room */
1092                 /* +2 for EOB chars. */
1093                 int number_to_move = (yy_n_chars) + 2;
1094                 char *dest = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[
1095                                         YY_CURRENT_BUFFER_LVALUE->yy_buf_size + 2];
1096                 char *source =
1097                                 &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move];
1098 
1099                 while ( source > YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
1100                         *--dest = *--source;
1101 
1102                 yy_cp += (int) (dest - source);
1103                 yy_bp += (int) (dest - source);
1104                 YY_CURRENT_BUFFER_LVALUE->yy_n_chars =
1105                         (yy_n_chars) = (int) YY_CURRENT_BUFFER_LVALUE->yy_buf_size;
1106 
1107                 if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 )
1108                         YY_FATAL_ERROR( "flex scanner push-back overflow" );
1109                 }
1110 
1111         *--yy_cp = (char) c;
1112 
1113         (yytext_ptr) = yy_bp;
1114         (yy_hold_char) = *yy_cp;
1115         (yy_c_buf_p) = yy_cp;
1116 }
1117 
1118 #endif
1119 
1120 #ifndef YY_NO_INPUT
1121 #ifdef __cplusplus
1122     static int yyinput (void)
1123 #else
1124     static int input  (void)
1125 #endif
1126 
1127 {
1128         int c;
1129     
1130         *(yy_c_buf_p) = (yy_hold_char);
1131 
1132         if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR )
1133                 {
1134                 /* yy_c_buf_p now points to the character we want to return.
1135                  * If this occurs *before* the EOB characters, then it's a
1136                  * valid NUL; if not, then we've hit the end of the buffer.
1137                  */
1138                 if ( (yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] )
1139                         /* This was really a NUL. */
1140                         *(yy_c_buf_p) = '\0';
1141 
1142                 else
1143                         { /* need more input */
1144                         int offset = (int) ((yy_c_buf_p) - (yytext_ptr));
1145                         ++(yy_c_buf_p);
1146 
1147                         switch ( yy_get_next_buffer(  ) )
1148                                 {
1149                                 case EOB_ACT_LAST_MATCH:
1150                                         /* This happens because yy_g_n_b()
1151                                          * sees that we've accumulated a
1152                                          * token and flags that we need to
1153                                          * try matching the token before
1154                                          * proceeding.  But for input(),
1155                                          * there's no matching to consider.
1156                                          * So convert the EOB_ACT_LAST_MATCH
1157                                          * to EOB_ACT_END_OF_FILE.
1158                                          */
1159 
1160                                         /* Reset buffer status. */
1161                                         yyrestart( yyin );
1162 
1163                                         /*FALLTHROUGH*/
1164 
1165                                 case EOB_ACT_END_OF_FILE:
1166                                         {
1167                                         if ( yywrap(  ) )
1168                                                 return 0;
1169 
1170                                         if ( ! (yy_did_buffer_switch_on_eof) )
1171                                                 YY_NEW_FILE;
1172 #ifdef __cplusplus
1173                                         return yyinput();
1174 #else
1175                                         return input();
1176 #endif
1177                                         }
1178 
1179                                 case EOB_ACT_CONTINUE_SCAN:
1180                                         (yy_c_buf_p) = (yytext_ptr) + offset;
1181                                         break;
1182                                 }
1183                         }
1184                 }
1185 
1186         c = *(unsigned char *) (yy_c_buf_p);    /* cast for 8-bit char's */
1187         *(yy_c_buf_p) = '\0';   /* preserve yytext */
1188         (yy_hold_char) = *++(yy_c_buf_p);
1189 
1190         return c;
1191 }
1192 #endif  /* ifndef YY_NO_INPUT */
1193 
1194 /** Immediately switch to a different input stream.
1195  * @param input_file A readable stream.
1196  * 
1197  * @note This function does not reset the start condition to @c INITIAL .
1198  */
1199     void yyrestart  (FILE * input_file )
1200 {
1201     
1202         if ( ! YY_CURRENT_BUFFER ){
1203         yyensure_buffer_stack ();
1204                 YY_CURRENT_BUFFER_LVALUE =
1205             yy_create_buffer( yyin, YY_BUF_SIZE );
1206         }
1207 
1208         yy_init_buffer( YY_CURRENT_BUFFER, input_file );
1209         yy_load_buffer_state(  );
1210 }
1211 
1212 /** Switch to a different input buffer.
1213  * @param new_buffer The new input buffer.
1214  * 
1215  */
1216     void yy_switch_to_buffer  (YY_BUFFER_STATE  new_buffer )
1217 {
1218     
1219         /* TODO. We should be able to replace this entire function body
1220          * with
1221          *              yypop_buffer_state();
1222          *              yypush_buffer_state(new_buffer);
1223      */
1224         yyensure_buffer_stack ();
1225         if ( YY_CURRENT_BUFFER == new_buffer )
1226                 return;
1227 
1228         if ( YY_CURRENT_BUFFER )
1229                 {
1230                 /* Flush out information for old buffer. */
1231                 *(yy_c_buf_p) = (yy_hold_char);
1232                 YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p);
1233                 YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars);
1234                 }
1235 
1236         YY_CURRENT_BUFFER_LVALUE = new_buffer;
1237         yy_load_buffer_state(  );
1238 
1239         /* We don't actually know whether we did this switch during
1240          * EOF (yywrap()) processing, but the only time this flag
1241          * is looked at is after yywrap() is called, so it's safe
1242          * to go ahead and always set it.
1243          */
1244         (yy_did_buffer_switch_on_eof) = 1;
1245 }
1246 
1247 static void yy_load_buffer_state  (void)
1248 {
1249         (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars;
1250         (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos;
1251         yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file;
1252         (yy_hold_char) = *(yy_c_buf_p);
1253 }
1254 
1255 /** Allocate and initialize an input buffer state.
1256  * @param file A readable stream.
1257  * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE.
1258  * 
1259  * @return the allocated buffer state.
1260  */
1261     YY_BUFFER_STATE yy_create_buffer  (FILE * file, int  size )
1262 {
1263         YY_BUFFER_STATE b;
1264     
1265         b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state )  );
1266         if ( ! b )
1267                 YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" );
1268 
1269         b->yy_buf_size = size;
1270 
1271         /* yy_ch_buf has to be 2 characters longer than the size given because
1272          * we need to put in 2 end-of-buffer characters.
1273          */
1274         b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2)  );
1275         if ( ! b->yy_ch_buf )
1276                 YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" );
1277 
1278         b->yy_is_our_buffer = 1;
1279 
1280         yy_init_buffer( b, file );
1281 
1282         return b;
1283 }
1284 
1285 /** Destroy the buffer.
1286  * @param b a buffer created with yy_create_buffer()
1287  * 
1288  */
1289     void yy_delete_buffer (YY_BUFFER_STATE  b )
1290 {
1291     
1292         if ( ! b )
1293                 return;
1294 
1295         if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */
1296                 YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0;
1297 
1298         if ( b->yy_is_our_buffer )
1299                 yyfree( (void *) b->yy_ch_buf  );
1300 
1301         yyfree( (void *) b  );
1302 }
1303 
1304 /* Initializes or reinitializes a buffer.
1305  * This function is sometimes called more than once on the same buffer,
1306  * such as during a yyrestart() or at EOF.
1307  */
1308     static void yy_init_buffer  (YY_BUFFER_STATE  b, FILE * file )
1309 
1310 {
1311         int oerrno = errno;
1312     
1313         yy_flush_buffer( b );
1314 
1315         b->yy_input_file = file;
1316         b->yy_fill_buffer = 1;
1317 
1318     /* If b is the current buffer, then yy_init_buffer was _probably_
1319      * called from yyrestart() or through yy_get_next_buffer.
1320      * In that case, we don't want to reset the lineno or column.
1321      */
1322     if (b != YY_CURRENT_BUFFER){
1323         b->yy_bs_lineno = 1;
1324         b->yy_bs_column = 0;
1325     }
1326 
1327         b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0;
1328     
1329         errno = oerrno;
1330 }
1331 
1332 /** Discard all buffered characters. On the next scan, YY_INPUT will be called.
1333  * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER.
1334  * 
1335  */
1336     void yy_flush_buffer (YY_BUFFER_STATE  b )
1337 {
1338         if ( ! b )
1339                 return;
1340 
1341         b->yy_n_chars = 0;
1342 
1343         /* We always need two end-of-buffer characters.  The first causes
1344          * a transition to the end-of-buffer state.  The second causes
1345          * a jam in that state.
1346          */
1347         b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR;
1348         b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR;
1349 
1350         b->yy_buf_pos = &b->yy_ch_buf[0];
1351 
1352         b->yy_at_bol = 1;
1353         b->yy_buffer_status = YY_BUFFER_NEW;
1354 
1355         if ( b == YY_CURRENT_BUFFER )
1356                 yy_load_buffer_state(  );
1357 }
1358 
1359 /** Pushes the new state onto the stack. The new state becomes
1360  *  the current state. This function will allocate the stack
1361  *  if necessary.
1362  *  @param new_buffer The new state.
1363  *  
1364  */
1365 void yypush_buffer_state (YY_BUFFER_STATE new_buffer )
1366 {
1367         if (new_buffer == NULL)
1368                 return;
1369 
1370         yyensure_buffer_stack();
1371 
1372         /* This block is copied from yy_switch_to_buffer. */
1373         if ( YY_CURRENT_BUFFER )
1374                 {
1375                 /* Flush out information for old buffer. */
1376                 *(yy_c_buf_p) = (yy_hold_char);
1377                 YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p);
1378                 YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars);
1379                 }
1380 
1381         /* Only push if top exists. Otherwise, replace top. */
1382         if (YY_CURRENT_BUFFER)
1383                 (yy_buffer_stack_top)++;
1384         YY_CURRENT_BUFFER_LVALUE = new_buffer;
1385 
1386         /* copied from yy_switch_to_buffer. */
1387         yy_load_buffer_state(  );
1388         (yy_did_buffer_switch_on_eof) = 1;
1389 }
1390 
1391 /** Removes and deletes the top of the stack, if present.
1392  *  The next element becomes the new top.
1393  *  
1394  */
1395 void yypop_buffer_state (void)
1396 {
1397         if (!YY_CURRENT_BUFFER)
1398                 return;
1399 
1400         yy_delete_buffer(YY_CURRENT_BUFFER );
1401         YY_CURRENT_BUFFER_LVALUE = NULL;
1402         if ((yy_buffer_stack_top) > 0)
1403                 --(yy_buffer_stack_top);
1404 
1405         if (YY_CURRENT_BUFFER) {
1406                 yy_load_buffer_state(  );
1407                 (yy_did_buffer_switch_on_eof) = 1;
1408         }
1409 }
1410 
1411 /* Allocates the stack if it does not exist.
1412  *  Guarantees space for at least one push.
1413  */
1414 static void yyensure_buffer_stack (void)
1415 {
1416         yy_size_t num_to_alloc;
1417     
1418         if (!(yy_buffer_stack)) {
1419 
1420                 /* First allocation is just for 2 elements, since we don't know if this
1421                  * scanner will even need a stack. We use 2 instead of 1 to avoid an
1422                  * immediate realloc on the next call.
1423          */
1424       num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */
1425                 (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc
1426                                                                 (num_to_alloc * sizeof(struct yy_buffer_state*)
1427                                                                 );
1428                 if ( ! (yy_buffer_stack) )
1429                         YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" );
1430 
1431                 memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*));
1432 
1433                 (yy_buffer_stack_max) = num_to_alloc;
1434                 (yy_buffer_stack_top) = 0;
1435                 return;
1436         }
1437 
1438         if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1){
1439 
1440                 /* Increase the buffer to prepare for a possible push. */
1441                 yy_size_t grow_size = 8 /* arbitrary grow size */;
1442 
1443                 num_to_alloc = (yy_buffer_stack_max) + grow_size;
1444                 (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc
1445                                                                 ((yy_buffer_stack),
1446                                                                 num_to_alloc * sizeof(struct yy_buffer_state*)
1447                                                                 );
1448                 if ( ! (yy_buffer_stack) )
1449                         YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" );
1450 
1451                 /* zero only the new slots.*/
1452                 memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*));
1453                 (yy_buffer_stack_max) = num_to_alloc;
1454         }
1455 }
1456 
1457 /** Setup the input buffer state to scan directly from a user-specified character buffer.
1458  * @param base the character buffer
1459  * @param size the size in bytes of the character buffer
1460  * 
1461  * @return the newly allocated buffer state object.
1462  */
1463 YY_BUFFER_STATE yy_scan_buffer  (char * base, yy_size_t  size )
1464 {
1465         YY_BUFFER_STATE b;
1466     
1467         if ( size < 2 ||
1468              base[size-2] != YY_END_OF_BUFFER_CHAR ||
1469              base[size-1] != YY_END_OF_BUFFER_CHAR )
1470                 /* They forgot to leave room for the EOB's. */
1471                 return NULL;
1472 
1473         b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state )  );
1474         if ( ! b )
1475                 YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" );
1476 
1477         b->yy_buf_size = (int) (size - 2);      /* "- 2" to take care of EOB's */
1478         b->yy_buf_pos = b->yy_ch_buf = base;
1479         b->yy_is_our_buffer = 0;
1480         b->yy_input_file = NULL;
1481         b->yy_n_chars = b->yy_buf_size;
1482         b->yy_is_interactive = 0;
1483         b->yy_at_bol = 1;
1484         b->yy_fill_buffer = 0;
1485         b->yy_buffer_status = YY_BUFFER_NEW;
1486 
1487         yy_switch_to_buffer( b  );
1488 
1489         return b;
1490 }
1491 
1492 /** Setup the input buffer state to scan a string. The next call to yylex() will
1493  * scan from a @e copy of @a str.
1494  * @param yystr a NUL-terminated string to scan
1495  * 
1496  * @return the newly allocated buffer state object.
1497  * @note If you want to scan bytes that may contain NUL values, then use
1498  *       yy_scan_bytes() instead.
1499  */
1500 YY_BUFFER_STATE yy_scan_string (const char * yystr )
1501 {
1502     
1503         return yy_scan_bytes( yystr, (int) strlen(yystr) );
1504 }
1505 
1506 /** Setup the input buffer state to scan the given bytes. The next call to yylex() will
1507  * scan from a @e copy of @a bytes.
1508  * @param yybytes the byte buffer to scan
1509  * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes.
1510  * 
1511  * @return the newly allocated buffer state object.
1512  */
1513 YY_BUFFER_STATE yy_scan_bytes  (const char * yybytes, int  _yybytes_len )
1514 {
1515         YY_BUFFER_STATE b;
1516         char *buf;
1517         yy_size_t n;
1518         int i;
1519     
1520         /* Get memory for full buffer, including space for trailing EOB's. */
1521         n = (yy_size_t) (_yybytes_len + 2);
1522         buf = (char *) yyalloc( n  );
1523         if ( ! buf )
1524                 YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" );
1525 
1526         for ( i = 0; i < _yybytes_len; ++i )
1527                 buf[i] = yybytes[i];
1528 
1529         buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR;
1530 
1531         b = yy_scan_buffer( buf, n );
1532         if ( ! b )
1533                 YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" );
1534 
1535         /* It's okay to grow etc. this buffer, and we should throw it
1536          * away when we're done.
1537          */
1538         b->yy_is_our_buffer = 1;
1539 
1540         return b;
1541 }
1542 
1543 #ifndef YY_EXIT_FAILURE
1544 #define YY_EXIT_FAILURE 2
1545 #endif
1546 
1547 static void yynoreturn yy_fatal_error (const char* msg )
1548 {
1549                         fprintf( stderr, "%s\n", msg );
1550         exit( YY_EXIT_FAILURE );
1551 }
1552 
1553 /* Redefine yyless() so it works in section 3 code. */
1554 
1555 #undef yyless
1556 #define yyless(n) \
1557         do \
1558                 { \
1559                 /* Undo effects of setting up yytext. */ \
1560         int yyless_macro_arg = (n); \
1561         YY_LESS_LINENO(yyless_macro_arg);\
1562                 yytext[yyleng] = (yy_hold_char); \
1563                 (yy_c_buf_p) = yytext + yyless_macro_arg; \
1564                 (yy_hold_char) = *(yy_c_buf_p); \
1565                 *(yy_c_buf_p) = '\0'; \
1566                 yyleng = yyless_macro_arg; \
1567                 } \
1568         while ( 0 )
1569 
1570 /* Accessor  methods (get/set functions) to struct members. */
1571 
1572 /** Get the current line number.
1573  * 
1574  */
1575 int yyget_lineno  (void)
1576 {
1577     
1578     return yylineno;
1579 }
1580 
1581 /** Get the input stream.
1582  * 
1583  */
1584 FILE *yyget_in  (void)
1585 {
1586         return yyin;
1587 }
1588 
1589 /** Get the output stream.
1590  * 
1591  */
1592 FILE *yyget_out  (void)
1593 {
1594         return yyout;
1595 }
1596 
1597 /** Get the length of the current token.
1598  * 
1599  */
1600 int yyget_leng  (void)
1601 {
1602         return yyleng;
1603 }
1604 
1605 /** Get the current token.
1606  * 
1607  */
1608 
1609 char *yyget_text  (void)
1610 {
1611         return yytext;
1612 }
1613 
1614 /** Set the current line number.
1615  * @param _line_number line number
1616  * 
1617  */
1618 void yyset_lineno (int  _line_number )
1619 {
1620     
1621     yylineno = _line_number;
1622 }
1623 
1624 /** Set the input stream. This does not discard the current
1625  * input buffer.
1626  * @param _in_str A readable stream.
1627  * 
1628  * @see yy_switch_to_buffer
1629  */
1630 void yyset_in (FILE *  _in_str )
1631 {
1632         yyin = _in_str ;
1633 }
1634 
1635 void yyset_out (FILE *  _out_str )
1636 {
1637         yyout = _out_str ;
1638 }
1639 
1640 int yyget_debug  (void)
1641 {
1642         return yy_flex_debug;
1643 }
1644 
1645 void yyset_debug (int  _bdebug )
1646 {
1647         yy_flex_debug = _bdebug ;
1648 }
1649 
1650 static int yy_init_globals (void)
1651 {
1652         /* Initialization is the same as for the non-reentrant scanner.
1653      * This function is called from yylex_destroy(), so don't allocate here.
1654      */
1655 
1656     (yy_buffer_stack) = NULL;
1657     (yy_buffer_stack_top) = 0;
1658     (yy_buffer_stack_max) = 0;
1659     (yy_c_buf_p) = NULL;
1660     (yy_init) = 0;
1661     (yy_start) = 0;
1662 
1663 /* Defined in main.c */
1664 #ifdef YY_STDINIT
1665     yyin = stdin;
1666     yyout = stdout;
1667 #else
1668     yyin = NULL;
1669     yyout = NULL;
1670 #endif
1671 
1672     /* For future reference: Set errno on error, since we are called by
1673      * yylex_init()
1674      */
1675     return 0;
1676 }
1677 
1678 /* yylex_destroy is for both reentrant and non-reentrant scanners. */
1679 int yylex_destroy  (void)
1680 {
1681     
1682     /* Pop the buffer stack, destroying each element. */
1683         while(YY_CURRENT_BUFFER){
1684                 yy_delete_buffer( YY_CURRENT_BUFFER  );
1685                 YY_CURRENT_BUFFER_LVALUE = NULL;
1686                 yypop_buffer_state();
1687         }
1688 
1689         /* Destroy the stack itself. */
1690         yyfree((yy_buffer_stack) );
1691         (yy_buffer_stack) = NULL;
1692 
1693     /* Reset the globals. This is important in a non-reentrant scanner so the next time
1694      * yylex() is called, initialization will occur. */
1695     yy_init_globals( );
1696 
1697     return 0;
1698 }
1699 
1700 /*
1701  * Internal utility routines.
1702  */
1703 
1704 #ifndef yytext_ptr
1705 static void yy_flex_strncpy (char* s1, const char * s2, int n )
1706 {
1707                 
1708         int i;
1709         for ( i = 0; i < n; ++i )
1710                 s1[i] = s2[i];
1711 }
1712 #endif
1713 
1714 #ifdef YY_NEED_STRLEN
1715 static int yy_flex_strlen (const char * s )
1716 {
1717         int n;
1718         for ( n = 0; s[n]; ++n )
1719                 ;
1720 
1721         return n;
1722 }
1723 #endif
1724 
1725 void *yyalloc (yy_size_t  size )
1726 {
1727                         return malloc(size);
1728 }
1729 
1730 void *yyrealloc  (void * ptr, yy_size_t  size )
1731 {
1732                 
1733         /* The cast to (char *) in the following accommodates both
1734          * implementations that use char* generic pointers, and those
1735          * that use void* generic pointers.  It works with the latter
1736          * because both ANSI C and C++ allow castless assignment from
1737          * any pointer type to void*, and deal with argument conversions
1738          * as though doing an assignment.
1739          */
1740         return realloc(ptr, size);
1741 }
1742 
1743 void yyfree (void * ptr )
1744 {
1745                         free( (char *) ptr );   /* see yyrealloc() for (char *) cast */
1746 }
1747 
1748 #define YYTABLES_NAME "yytables"
1749 
1750 #line 6 "01.count_words.l"
1751 
1752 
1753 int main(){ 
1754   yylex();
1755   printf("\n");
1756   return 0;
1757 }

Count words(I)

1 %%
2 [0-9]+                 {printf("INT:%s\n",yytext);}
3 [a-zA-Z_][a-zA-Z_0-9]* {printf("ID :%s\n",yytext);}
4 .   ;
5 \n  ;
6 %%
7 
8 int main(){ 
9   yylex();
10   printf("\n");
11   return 0;
12 }
1 aDDDaaHHbm412abAmns12bm2
2 aabbbbbb2341abcccccc
3 79aaaaabbbbbm8888888882
4 ab6669bm8888884
5 ab6669bm888888
6 abbaldkjA
7 1ab 238234234 2342.234234 34 3.3
8 abm 9bm88888
9 66abm
 1 ID :aDDDaaHHbm412abAmns12bm2
 2 ID :aabbbbbb2341abcccccc
 3 INT:79
 4 ID :aaaaabbbbbm8888888882
 5 ID :ab6669bm8888884
 6 ID :ab6669bm888888
 7 ID :abbaldkjA
 8 INT:1
 9 ID :ab
10 INT:238234234
11 INT:2342
12 INT:234234
13 INT:34
14 INT:3
15 INT:3
16 ID :abm
17 INT:9
18 ID :bm88888
19 INT:66
20 ID :abm

Count words(II)

1 %%
2 [0-9]+                 {printf("int:%s\n",yytext);}
3 [a-zA-Z_][a-zA-Z_0-9]+ {printf("ID :%s\n",yytext);}
4 [1-4]+abc*             {printf("2  :%s\n",yytext);}
5 .   ;
6 \n  ;
7 %%
8 int main(){ 
9   yylex();
10   printf("\n");
11   return 0;
12 }
1 aDDDaaHHbm412abAmns12bm2
2 aabbbbbb2341abcccccc
3 79aaaaabbbbbm8888888882
4 ab6669bm8888884
5 ab6669bm888888
6 abbaldkjA
7 1ab 238234234 2342.234234 34 3.3
8 abm 9bm88888
9 66abm
 1 lex$ ./a.out < count_words_input_1.txt
 2 
 3 ID :aDDDaaHHbm412abAmns12bm2
 4 ID :aabbbbbb2341abcccccc
 5 int:79
 6 ID :aaaaabbbbbm8888888882
 7 ID :ab6669bm8888884
 8 ID :ab6669bm888888
 9 ID :abbaldkjA
10 2  :1ab
11 int:238234234
12 int:2342
13 int:234234
14 int:34
15 int:3
16 int:3
17 ID :abm
18 int:9
19 ID :bm88888
20 int:66
21 ID :abm

Count words(III)-input(I)

1 %%
2 ^[0-9]+a+b+          {printf("1 %s\n", yytext);}
3 [1-4]+abc*           {printf("2 %s\n", yytext);}
4 [a-zA-Z]*[4-5]+      {printf("3 %s\n", yytext);}
5 [^0-8]bm8*$          {printf("4 %s\n", yytext);}
6 .                       ;
7 \n                      ;
8 %%
9 int main(){ yylex();printf("\n");return 0;}
1 aDDDaaHHbm412abAmns12bm2
2 aabbbbbb2341abcccccc
3 79aaaaabbbbbm8888888882
4 ab6669bm8888884
5 ab6669bm888888
6 abbaldkjA
7 1ab 238234234 2342.234234 34 3.3
8 abm 9bm88888
9 66abm
 1 lex$ flex 04.count_words.l
 2 lex$ gcc lex.yy.c -lfl
 3 lex$ ./a.out < count_words_input_1.txt
 4 
 5 3 aDDDaaHHbm4
 6 2 12ab
 7 2 2341abcccccc
 8 1 79aaaaabbbbb
 9 3 4
10 4 9bm888888
11 1 1ab
12 3 4
13 3 4
14 3 4
15 3 4
16 3 4
17 3 4
18 4 9bm88888
19 1 66ab

Count words(III)-input(II)

1 %%
2 ^[0-9]+a+b+          {printf("1 %s\n", yytext);}
3 [1-4]+abc*           {printf("2 %s\n", yytext);}
4 [a-zA-Z]*[4-5]+      {printf("3 %s\n", yytext);}
5 [^0-8]bm8*$          {printf("4 %s\n", yytext);}
6 .                       ;
7 \n                      ;
8 %%
9 int main(){ yylex();printf("\n");return 0;}
1 aDDDaaHHbm412abAmns12bm2
2 aabbbbbb2341abcccccc
3 79aaaaabbbbbm8888888882
4 ab6669bm8888884
5 ab6669bm888888
6 abbaldkjA
7 1ab 238234234 2342.234234 34 3.3
8 abm 9bm88888
9 66abm
10 
 1 lex$ flex 04.count_words.l
 2 lex$ gcc lex.yy.c -lfl
 3 lex$ ./a.out < count_words_input_2.txt
 4 
 5 3 aDDDaaHHbm4
 6 2 12ab
 7 2 2341abcccccc
 8 1 79aaaaabbbbb
 9 3 4
10 4 9bm888888
11 1 1ab
12 3 4
13 3 4
14 3 4
15 3 4
16 3 4
17 3 4
18 4 9bm88888
19 1 66ab

Simple Number

1 %%
2 ^aa             {printf("AAAAAAA\n"); }
3 ^ss             {printf("SSSSSSS\n"); }
4 [0-9]+          {printf("Integer:: %s\n",yytext);}
5 [0-9]+\.[0-9]+   {printf("Real :: %s\n",yytext);}
6 .                       ;
7 \n                      ;
8 %%
9 int main(){ 
10     yylex();
11     printf("\n");
12     return 0;
13 }
1 12 234 434.45 676.45
2 eer
3 df
4 aa   abc bb aa
5 ss aa aaa  ss abcabc  654  sdfsf 123 lsdkjfsl 
6 aa   ss
 1 lex$ flex 05.simpleNumber.l
 2 lex$ gcc lex.yy.c -lfl
 3 lex$ ./a.out < 05.simpleNumber_input.txt
 4 
 5 Integer:: 12
 6 Integer:: 234
 7 Real :: 434.45
 8 Real :: 676.45
 9 AAAAAAA
10 SSSSSSS
11 Integer:: 654
12 Integer:: 123
13 AAAAAAA

Number of Characters

1 %{
2 int numberOfInt=0   , numberOfDouble=0 , 
3   numberOfString = 0 ,  numberOfLine =0 ;
4 int numberOfDot =0 , numberOfOther=0;
5 %}
6 %%
7 \n              {numberOfLine++;}
8 [0-9]+          {numberOfInt++;}
9 [0-9]*\.[0-9]+  {numberOfDouble++;}
10 \.              {numberOfDot++;}
11 [a-zA-Z]+       {numberOfString++;}
12 [ \t]           {}
13 .               {numberOfOther++;}
14 %%
15 int main(){
16   yylex();
17   printf("Int::%d\n", numberOfInt);
18   printf("Double::%d\n", numberOfDouble);
19   printf("Double::%d\n", numberOfString);
20   printf("Line::%d\n", numberOfLine);
21   printf("Dot::%d\n", numberOfDot);
22   printf("Other::%d\n", numberOfOther);
23 }
1 aaabm12Amns12bm8
2 aabbbbbb2345abcccccc
3 79bm8888888882
4 ab666bm888888
5 bm9abbbbbb88abbbbbb
6 bm
 1 lex$ flex 08.numberOfCharacters.l
 2 lex$ gcc lex.yy.c -lfl
 3 lex$ ./a.out < 08.numberOfCharacters_input.txt
 4 
 5 Int::10
 6 Double::0
 7 Double::12
 8 Line::6
 9 Dot::0
10 Other::0

Parts of Speech(I)

1 %%
2 [\t ]+   {/* ignore white space */ ;}
3 aaa|bb {printf("%s :::: mytest\n",yytext);}
4 is|am|are|were|was|be {printf("%s: is a verb\n", yytext);}
5 being|been|do|does  {printf("%s: is a verb\n", yytext);}
6 did|will|would|should {printf("%s: is a verb\n", yytext);}
7 can|could|has|have {printf("%s: is a verb\n", yytext);}
8 had|go  {printf("%s: is a verb\n", yytext);}
9 very|simply|gently {printf("%s: is an adverb\n", yytext);}
10 quietly|calmly|angrily {printf("%s: is an adverb\n",yytext);}
11 to|from|behind { printf("%s: is a preposition\n", yytext);}
12 above|below  {printf("%s: is a preposition\n", yytext);}
13 between|below {printf("%s: is a preposition\n", yytext);}
14 if|then|and {printf("%s: is a conjunction\n", yytext);}
15 but|or {printf("%s: is a conjunction\n", yytext);}
16 their|my|your {printf("%s: is an adjective\n", yytext);}
17 his|her|its  {printf("%s: is an adjective\n", yytext);}
18 I|you|he|she {printf("%s: in a pronoun\n", yytext);}
19 we|they {printf("%s: in a pronoun\n", yytext);}
20 [a-zA-Z]+ {printf("%s: don't recognize, a noun\n",yytext);}
21 \&.|\n    { ECHO; /* normal default anyway */ }
22 %%
23 int main(){ yylex(); }
1 aaa bbb is  am above
2 I you 2376429734 2342.23423
 1 aaa :::: mytest
 2 bbb: don't recognize, a noun
 3 is: is a verb
 4 am: is a verb
 5 above: is a preposition
 6 
 7 I: in a pronoun
 8 you: in a pronoun
 9 23764297342342.23423

Parts of Speech(II)

1 verb am
2 verb is
3 verb are
4 prep at
5 prep of
6 prep in
7 I am an instructor.
8 prep to
9 noun orange window
10 verb have has
11 noun university Kashan Ahmad
12 They have a situation.
13 This orange is not red.
14 pron I  you
15 My name is Ahmad.
16 I'm an instructor at 
17 the university of Kashan.
18 You are students.
 1 I:don't recognize
 2 am:verb
 3 an:don't recognize
 4 instructor:don't recognize
 5 They:don't recognize
 6 have:verb
 7 a:don't recognize
 8 situation:don't recognize
 9 This:don't recognize
10 orange:noun
11 is:verb
12 not:don't recognize
13 red:don't recognize
14 My:don't recognize
15 name:don't recognize
16 is:verb
17 Ahmad:noun
18 I:pronoun
19 m:don't recognize
20 an:don't recognize
21 instructor:don't recognize
22 at:preposition
23 the:don't recognize
24 university:noun
25 of:preposition
26 Kashan:noun
27 You:don't recognize
28 are:verb
29 students:don't recognize

Parts of Speech(III)

1 %{/* Word recognizer with a symbol table.*/
2 enum{
3   LOOKUP = 0,
4   /*default-looking rather than defining.*/
5   VERB,ADJ,ADV,NOUN,PREP,PRON,CONJ, ART
6 };
7 int state=LOOKUP;
8 int add_word(int type, char *word);
9 int lookup_word(char *word);
10 %}
11 %%
12 \n {
13   state = LOOKUP;
14 }/* end of line, return to default state */
15 ^verb { state = VERB; }
16 ^adj  { state = ADJ; }
17 ^adv  { state = ADV; }
18 ^noun { state = NOUN; }
19 ^prep { state = PREP; }
20 ^pron { state = PRON; }
21 ^conj { state = CONJ; }
22 ^article {state = ART; }
23 [a-zA-Z]+  {
24   /*a normal word, define it or look it up*/
25   if(state != LOOKUP)
26   /* define the current word */
27     add_word(state, yytext);
28   else
1     switch(lookup_word(yytext)){
2       case VERB:
3         printf("%s:verb\n", yytext); break;
4       case ADJ:
5         printf("%s:adjective\n", yytext);
6         break;
7       case ADV:
8         printf("%s:adverb\n", yytext); 
9         break;
10       case NOUN:
11         printf("%s:noun\n", yytext); break;
12       case PREP:
13         printf("%s:preposition\n",yytext);
14         break;
15       case PRON:
16         printf("%s:pronoun\n", yytext);
17         break;
18       case CONJ:
19         printf("%s:conjunction\n", yytext);
20         break;
21       case ART:
22         printf("%s:article\n", yytext); 
23         break;
24       default:
25         printf("%s",yytext);
26         printf("don't recognize\n");
27     }
28 }      /* [a-zA-Z]+  {  */

Parts of Speech(IV)

1 .  /* ignore anything else */ ;
2 %%
3 int main(){yylex();}
4 /*define a linked list of words and types*/
5 struct word{
6   char *word_name; 
7   int word_type;
8   struct word *next;
9 };
10 
11 /*first element in word list*/
12 struct word *word_list;
13 
14 extern void *malloc();
15 
16 int add_word(int type, char *word){
17   struct word *wp;
18   if(lookup_word(word) != LOOKUP){
19     printf("!!! warning: ");
20     printf("%s already defined\n", word);
21     return 0;
22   }
23   /* word not there, allocate a new entry
24      and link it on the list
1   */
2   wp = (struct word *)
3     malloc(sizeof(struct word));
4 
5   wp->next = word_list;
6   /* have to copy the word itself as well*/
7 
8   wp->word_name = (char *)
9     malloc(strlen(word)+1);
10 
11   strcpy(wp->word_name, word);
12   wp->word_type = type;
13   word_list = wp;return 1;/* it worked */
14 }
15 
16 int  lookup_word(char *word){
17   struct word *wp = word_list;/*
18   search down the list looking for the word
19   */
20   for(; wp; wp = wp->next)
21     if(strcmp(wp->word_name, word) == 0)
22       return wp->word_type;
23   return LOOKUP;    /* not found */
24 }

Some References

Similar Lexer and Parser

  1. PLY
  2. Lemon for SQLite
    1. https://www.sqlite.org/lang.html
    2. https://www.sqlite.org/pragma.html

PLY / lex part

Sample I

1 tokens = ('NUMBER','PLUS', 'MUL', 'ID')
2 t_PLUS    = r'\+'
3 t_MUL    = r'\*'
4 t_ID = r'[a-zA-Z_][a-zA-Z_0-9]*'
5 t_NUMBER = r'[0-9]+'
6 
7 t_ignore = " \t"
8 
9 import ply.lex as lex
10 lex.lex()
11 
12 lex.input("5434+2882 + count34+   45")
13 while True:
14   tok = lex.token()
15   if not tok: break
16   print(tok)

Sample II

1 tokens = ('NUMBER','PLUS')
2 t_PLUS    = r'\+'
3 
4 def t_NUMBER(t):
5   r'[0-9]+'
6   t.value = int(t.value)
7   return t
8 
9 t_ignore = " \t"
10 
11 def t_error(t):
12   print("Illegal character '%s'" % t.value[0])
13   t.lexer.skip(1)
14 
15 import ply.lex as lex
16 lex.lex()
17 lex.input("*4+2")
18 while True:
19   tok = lex.token()
20   if not tok:  break
21   print(tok)

Sample III

1 tokens = ('NUMBER','PLUS')
2 t_PLUS    = r'\+'
3 
4 def t_NUMBER(t):
5   r'[0-9]+'
6   t.value = int(t.value)
7   return t
8 
9 t_ignore = " \t"
10 
11 def t_error(t):
12     print("Illegal character '%s'" % t.value[0])
13     t.lexer.skip(1)
14 
15 import ply.lex as lex
16 lex.lex()
17 lex.input("*4+2")
18 while True:
19   tok = lex.token()
20   if not tok:  break
21   print(tok)
22 """
23 lex$ python3 126.error.plus.py 
24 Illegal character '*'
25 LexToken(NUMBER,4,1,1)
26 LexToken(PLUS,'+',1,2)
27 LexToken(NUMBER,2,1,3)
28 lex$ 
29 """

Sample IV

1 # 128.pluse.mul.py
2 tokens = ('NUMBER','PLUS', 'MUL')
3 t_PLUS    = r'\+'
4 t_MUL    = r'\*'
5 
6 def t_NUMBER(t):
7   r'[0-9]+'
8   t.value = int(t.value)
9   return t
10 
11 t_ignore = " \t"
12 
13 def t_error(t):
14     print("Illegal character '%s'" % t.value[0])
15     t.lexer.skip(1)
16 
17 import ply.lex as lex
18 lex.lex()
19 lex.input("*445+23*34+123")
20 
21 while True:
22   tok = lex.token()
23   if not tok:  break
24   print(tok)
25 
26 """
27 lex$ python3 128.plus.mul.py 
28 LexToken(MUL,'*',1,0)
29 LexToken(NUMBER,445,1,1)
30 LexToken(PLUS,'+',1,4)
31 LexToken(NUMBER,23,1,5)
32 LexToken(MUL,'*',1,7)
33 LexToken(NUMBER,34,1,8)
34 LexToken(PLUS,'+',1,10)
35 LexToken(NUMBER,123,1,11)
36 lex$ 
37 """

Sample V

1 tokens = ('NUMBER','PLUS', 'MUL', 'ID')
2 t_PLUS    = r'\+'
3 t_MUL    = r'\*'
4 t_ID = r'[a-zA-Z_][a-zA-Z_0-9]*'
5 
6 def t_NUMBER(t):
7   r'[0-9]+'
8   t.value = int(t.value)
9   return t
10 
11 t_ignore = " \t"
12 
13 def t_error(t):
14     print("Illegal character '%s'" % t.value[0])
15     t.lexer.skip(1)
16 
17 import ply.lex as lex
18 lex.lex()
19 lex.input("wwe2+4+23*34+123")
21 while True:
22   tok = lex.token()
23   if not tok:  break
24   print(tok)
25 """
26 lex$ python3 130.id.py 
27 LexToken(ID,'wwe2',1,0)
28 LexToken(PLUS,'+',1,4)
29 LexToken(NUMBER,4,1,5)
30 LexToken(PLUS,'+',1,6)
31 LexToken(NUMBER,23,1,7)
32 LexToken(MUL,'*',1,9)
33 LexToken(NUMBER,34,1,10)
34 LexToken(PLUS,'+',1,12)
35 LexToken(NUMBER,123,1,13)
36 lex$ 
37 """

Sample VI

1 tokens = ('NUMBER','PLUS', 'MUL', 'ID', 'ASG')
2 t_PLUS    = r'\+'
3 t_MUL    = r'\*'
4 t_ID = r'[a-zA-Z_][a-zA-Z_0-9]*'
5 t_ASG = '='
6 t_NUMBER = r'[0-9]+'
7 t_ignore = " \t\n"
8 
9 def t_error(t):
10     print("Illegal character '%s'" % t.value[0])
11     t.lexer.skip(1)
12 
13 import ply.lex as lex
14 lex.lex()
15 lex.input("""wwe2 = 4+23 *34 +123
16 count+34+435
17 x=y
18 """)
19 while True:
20   tok = lex.token()
21   if not tok:  break
22   print(tok)
24 """
25 lex$ python3 132.assign.py
26 LexToken(ID,'wwe2',1,0)
27 LexToken(ASG,'=',1,5)
28 LexToken(NUMBER,4,1,7)
29 LexToken(PLUS,'+',1,8)
30 LexToken(NUMBER,23,1,9)
31 LexToken(MUL,'*',1,12)
32 LexToken(NUMBER,34,1,13)
33 LexToken(PLUS,'+',1,16)
34 LexToken(NUMBER,123,1,17)
35 LexToken(ID,'count',1,21)
36 LexToken(PLUS,'+',1,26)
37 LexToken(NUMBER,34,1,27)
38 LexToken(PLUS,'+',1,29)
39 LexToken(NUMBER,435,1,30)
40 LexToken(ID,'x',1,34)
41 LexToken(ASG,'=',1,35)
42 LexToken(ID,'y',1,36)
43 lex$
44 """

Sample VII

1 tokens = ('NUMBER','PLUS', 'MUL', 'ID', 'ASG',
2   'COMP')
3 t_PLUS    = r'\+'
4 t_MUL    = r'\*'
5 t_ID = r'[a-zA-Z_][a-zA-Z_0-9]*'
6 t_ASG = '='
7 t_COMP = '=='
8 t_NUMBER = r'[0-9]+'
9 t_ignore = " \t\n"
10 
11 def t_error(t):
12     print("Illegal rrr '%s'" % t.value[0])
13     t.lexer.skip(1)
14 
15 import ply.lex as lex
16 lex.lex()
17 lex.input("""wwe2 ==  4+23 = *34 +123
18 count+34+435
19 x=y
20 """)
22 while True:
23   tok = lex.token()
24   if not tok:  break
25   print(tok)
26 
27 """
28 lex$ python3 132.assign.py 
29 LexToken(ID,'wwe2',1,0)
30 LexToken(ASG,'=',1,5)
31 LexToken(NUMBER,4,1,7)
32 LexToken(PLUS,'+',1,8)
33 LexToken(NUMBER,23,1,9)
34 LexToken(MUL,'*',1,12)
35 LexToken(NUMBER,34,1,13)
36 LexToken(PLUS,'+',1,16)
37 LexToken(NUMBER,123,1,17)
38 LexToken(ID,'count',1,21)
39 LexToken(PLUS,'+',1,26)
40 LexToken(NUMBER,34,1,27)
41 LexToken(PLUS,'+',1,29)
42 LexToken(NUMBER,435,1,30)
43 LexToken(ID,'x',1,34)
44 LexToken(ASG,'=',1,35)
45 LexToken(ID,'y',1,36)
46 lex$ 
47 """

Sample VIII

1 tokens = ('NUMBER','PLUS', 'MUL', 'ID', 'ASG')
2 t_PLUS    = r'\+'
3 t_MUL    = r'\*'
4 t_ID = r'[a-zA-Z_][a-zA-Z_0-9]*'
5 t_ASG = '='
6 
7 def t_NUMBER(t):
8   r'[0-9]+'
9   t.value = int(t.value)
10   return t
11 
12 t_ignore = " \t"
13 
14 def t_newline(t):
15   r'\n+'
16   t.lexer.lineno += len(t.value)
17   print('new line')
18 
19 def t_error(t):
20     print("Illegal character '%s'" % t.value[0])
21     t.lexer.skip(1)
22 
23 import ply.lex as lex
24 lex.lex()
25 lex.input("""wwe2 = 4+23 *34 +123
26 count+34+435
27 x=y
28 """)
29 while True:
30   tok = lex.token()
31   if not tok:  break
32   print(tok)
33 """
34 lex$ python3 134.newline.py 
35 LexToken(ID,'wwe2',1,0)
36 LexToken(ASG,'=',1,5)
37 LexToken(NUMBER,4,1,7)
38 LexToken(PLUS,'+',1,8)
39 LexToken(NUMBER,23,1,9)
40 LexToken(MUL,'*',1,12)
41 LexToken(NUMBER,34,1,13)
42 LexToken(PLUS,'+',1,16)
43 LexToken(NUMBER,123,1,17)
44 new line
45 LexToken(ID,'count',2,21)
46 LexToken(PLUS,'+',2,26)
47 LexToken(NUMBER,34,2,27)
48 LexToken(PLUS,'+',2,29)
49 LexToken(NUMBER,435,2,30)
50 new line
51 LexToken(ID,'x',3,34)
52 LexToken(ASG,'=',3,35)
53 LexToken(ID,'y',3,36)
54 new line
55 """

A calculator

1 tokens = ('NUMBER','PLUS')
2 t_PLUS    = r'\+'
3 
4 def t_NUMBER(t):
5   r'[0-9]+'
6   t.value = int(t.value)
7   return t
8 
9 t_ignore = " \t"
10 
11 def t_error(t):
12   print("Illegal character '%s'" % t.value[0])
13   t.lexer.skip(1)
14 
15 import ply.lex as lex
16 lex.lex()
17 
18 """
19 S → E
20 E → E + T
21 E → a
22 """
23 def p_s(p):
24   'S : E' 
25   print('S → E ', p[1])
26 
27 def p_e_e_a(p):
28   'E : E PLUS NUMBER' # E → E + a
29   p[0] = p[1] + p[3]
30   print('E → E_1 + a :', 'E: ', end='')
31   print(p[0], ' \t E1: ', p[1], end='')
32   print('\t a: ', p[3])
33 
34 def p_e_a(p):
35   'E : NUMBER' # E → a
36   p[0] = p[1]
37   print('E → a :', p[1])
38 
39 def p_error(p):
40   print("Syntax error at '%s'" % p.value)
41 
42 import ply.yacc as yacc;
43 yacc.yacc()
44 
45 while True:
46   s = input('calc > ')
47   if s.strip()=='':break
48   yacc.parse(s)

END

1