/****************************************************************/
/*                                                              */
/*      Program 2-1:  Simple Tokenizer                          */
/*                                                              */
/*      Recognize words, small integers, and the period.        */
/*                                                              */
/*      FILE:       token1.c                                    */
/*                                                              */
/*      USAGE:      token1 sourcefile                           */
/*                                                              */
/*          sourcefile      name of source file to tokenize     */
/*                                                              */
/*      Copyright (c) 1991 by Ronald Mak                        */
/*      For instructional purposes only.  No warranties.        */
/*                                                              */
/*	Modified by M.Auguston	, 8/8/96			*/
/*      Modified by B.Margolis  , 9/1/98                        */
/*      Modified by D.Tappan    , 8/9/99                        */
/*      Modified by M.Michie    , 2/11/99                       */
/****************************************************************/

#include <stdio.h>
#include <math.h>
#include <sys/types.h>
#include <sys/times.h>

#define FORM_FEED_CHAR          '\f'
#define EOF_CHAR                '\x7f'
#define COMMENT_CHAR            '/'
#define STRING_CHAR             '\''

#define MAX_FILE_NAME_LENGTH    32
#define MAX_SOURCE_LINE_LENGTH  256
#define MAX_PRINT_LINE_LENGTH   80
#define MAX_LINES_PER_PAGE	50
#define DATE_STRING_LENGTH      26
#define MAX_TOKEN_STRING_LENGTH MAX_SOURCE_LINE_LENGTH
#define MAX_CODE_BUFFER_SIZE    4096

#define MAX_INTEGER             32767
#define MAX_DIGIT_COUNT         20

typedef enum {
    FALSE, TRUE
} BOOLEAN;

/*--------------------------------------------------------------*/
/*  Character codes                                             */
/*--------------------------------------------------------------*/

typedef enum {
  LETTER, DIGIT, SPECIAL, EOF_CODE, COMMENT_CODE, STRING_CODE
} CHAR_CODE;

/*--------------------------------------------------------------*/
/*  Token codes							*/
/*--------------------------------------------------------------*/

typedef enum {
    NO_TOKEN, WORD, NUMBER, PERIOD,
    END_OF_FILE, ERROR, RESERVED, COMMENT, STRING
} TOKEN_CODE;

/*--------------------------------------------------------------*/
/*  Token name strings                                          */
/*--------------------------------------------------------------*/

char *symbol_strings[] = {
    "<no token>", "<WORD>", "<NUMBER>", "<PERIOD>",
    "<END OF FILE>", "<ERROR>", "<RESERVED>", "<COMMENT>", "<STRING>"
};

/*--------------------------------------------------------------*/
/*  Literal structure                                           */
/*--------------------------------------------------------------*/

typedef enum {
    INTEGER_LIT, STRING_LIT
} LITERAL_TYPE;

typedef struct {
    LITERAL_TYPE type;
    union {
	int  integer;
	char string[MAX_SOURCE_LINE_LENGTH];
    } value;
} LITERAL;

/*--------------------------------------------------------------*/
/*  Globals                                                     */
/*--------------------------------------------------------------*/

char        ch;                 /* current input character */
char        nextch;             /* read ahead character */
TOKEN_CODE  token;              /* code of current token */
LITERAL     literal;            /* value of literal */
int         buffer_offset;      /* char offset into source buffer */
int         line_number = 0;    /* current line number */
int         out_number = 0;

BOOLEAN     newline = FALSE;
char source_buffer[MAX_SOURCE_LINE_LENGTH]; /* source file buffer */
char token_string[MAX_TOKEN_STRING_LENGTH]; /* token string */
char *bufferp = source_buffer;              /* source buffer ptr */
char *tokenp  = token_string;               /* token string ptr */

int     digit_count;            /* total no. of digits in number */
BOOLEAN count_error;            /* too many digits in number? */

int page_number = 0;
int line_count  = MAX_LINES_PER_PAGE;   /* no. lines on current pg */

char source_name[MAX_FILE_NAME_LENGTH]; /* name of source file */
char date[DATE_STRING_LENGTH];          /* current date and time */

FILE *source_file;

CHAR_CODE char_table[256];

/*--------------------------------------------------------------*/
/*  Function Prototypes                                         */
/*--------------------------------------------------------------*/
void    print_token();
void    init_scanner(char *name);
void    quit_scanner();
void    get_char();
void    skip_blanks();
void    get_token();
void    get_word();
void    get_comment();
void    get_string();
void    get_number();
void    get_special();
void    open_source_file(char *name);
void    close_source_file();
BOOLEAN get_source_line();
void    print_line(char line[]);
void    init_page_header(char *name);
void    print_page_header();

/*--------------------------------------------------------------*/
/*  char_code           Return the character code of ch.        */
/*--------------------------------------------------------------*/

#define char_code(ch)   char_table[ch]

/*--------------------------------------------------------------*/
/*  main                Loop to tokenize source file.           */
/*--------------------------------------------------------------*/

main(int argc, char *argv[])
{
    /*
    --  Initialize the scanner.
    */
    init_scanner(argv[1]);

    /*
    --  Repeatedly fetch tokens until a period
    --  or the end of file.
    */
    do {
	get_token();
	if (token == END_OF_FILE) {
	    print_line("*** ERROR: Unexpected end of file.\n");
	    break;
	}

	print_token();
    } while (token != PERIOD);

    quit_scanner();
}

/*--------------------------------------------------------------*/
/*  print_token         Print a line describing the current     */
/*                      token.                                  */
/*--------------------------------------------------------------*/
void print_token()
{
    char line[MAX_PRINT_LINE_LENGTH];
    char *symbol_string = symbol_strings[token];
    
    if (token != COMMENT)
    switch (token) {
	case NUMBER:
	    sprintf(line, "     >> %-16s %d\n",
			  symbol_string, literal.value.integer);
            break;

	default:
	    sprintf(line, "     >> %-16s %-s\n",
			  symbol_string, token_string);
	    break;
    }

    print_line(line);
}

		/********************************/
		/*				*/
		/*	Initialization		*/
		/*				*/
		/********************************/

/*--------------------------------------------------------------*/
/*  init_scanner	Initialize the scanner globals		*/
/*			and open the source file.		*/
/*                                                              */
/*  Parameters:         name - name of source file              */
/*--------------------------------------------------------------*/

void init_scanner(char *name)
{
    int ch;

    /*
    --  Initialize character table.
    */
    for (ch = 0;   ch < 256;  ++ch) char_table[ch] = SPECIAL;
    for (ch = '0'; ch <= '9'; ++ch) char_table[ch] = DIGIT;
    for (ch = 'A'; ch <= 'Z'; ++ch) char_table[ch] = LETTER;
    for (ch = 'a'; ch <= 'z'; ++ch) char_table[ch] = LETTER;

    char_table[EOF_CHAR] = EOF_CODE;
    char_table[COMMENT_CHAR] = COMMENT_CODE;
    char_table[STRING_CHAR] = STRING_CODE;

    init_page_header(name);
    open_source_file(name);
}

/*--------------------------------------------------------------*/
/*  quit_scanner	Terminate the scanner.			*/
/*--------------------------------------------------------------*/
void quit_scanner()
{
    close_source_file();
}

		/********************************/
		/*				*/
		/*	Character routines	*/
		/*				*/
		/********************************/

/*--------------------------------------------------------------*/
/*  get_char		Set ch to the next character from the	*/
/*			source buffer.				*/
/*--------------------------------------------------------------*/
void get_char()
{
    BOOLEAN get_source_line();

    /*
    --  If at end of current source line, read another line.
    --  If at end of file, set ch to the EOF character and return.
    */
    if (*bufferp == '\0') {
	if (! get_source_line()) {
	    ch = EOF_CHAR;
	    return;
	}
	bufferp = source_buffer;
	buffer_offset = 0;
    }

    ch = *bufferp++;    /* next character in the buffer */
    nextch = *(bufferp + 1);

    if (ch == '\n') {
      newline = TRUE;
    }
    else
      newline = FALSE;

    if ((ch == '\n') || (ch == '\t')) ch = ' ';
    
}

/*--------------------------------------------------------------*/
/*  skip_blanks		Skip past any blanks at the current	*/
/*			location in the source buffer.  Set	*/
/*			ch to the next nonblank character.	*/
/*--------------------------------------------------------------*/
void skip_blanks()
{
    while (ch == ' ') get_char();
}

		/********************************/
		/*				*/
		/*	Token routines		*/
		/*				*/
		/********************************/

	/*  Note that after a token has been extracted, */
	/*  ch is the first character after the token.  */

/*--------------------------------------------------------------*/
/*  get_token		Extract the next token from the	source	*/
/*			buffer.					*/
/*--------------------------------------------------------------*/
void get_token()
{
  skip_blanks();
  tokenp = token_string;
  
  switch (char_code(ch)) {
  case COMMENT_CODE:
    get_comment();
    break;
  case STRING_CODE:
    get_string();
    break;
  case LETTER:    
    get_word();             
    break;
  case DIGIT:     
    get_number();           
    break;
  case EOF_CODE:  
    token = END_OF_FILE;    
    break;
  default:        
    get_special();          
      break;
  }
}

/*--------------------------------------------------------------*/
/*  get_string          Extract a string                        */
/*--------------------------------------------------------------*/
void get_string ()
{

  do {
    if (nextch == '\'') {
      *tokenp++ = nextch;
    }
    *tokenp++ = ch;
    get_char();
  } while (ch != '\'' || newline == TRUE);

  get_char();

  *tokenp = '\0';

  token = STRING;
}
/*--------------------------------------------------------------*/
/*  get_comment         Extract a comment.                      */
/*--------------------------------------------------------------*/
void get_comment ()
{    
  get_char();
  if (ch == '/') {
    while (newline != TRUE) 
      get_char();

    *tokenp = '\0';
    *token_string = '\0';

    token = COMMENT;
  }
  else 
    token = ERROR;
}

/*--------------------------------------------------------------*/
/*  is_reserved_word    Extract a word and test if it is one    */
/*                      of the reserved words                   */
/*--------------------------------------------------------------*/
BOOLEAN is_reserved_word()
{
  int i = 0;
  char tmp_string [MAX_TOKEN_STRING_LENGTH]; /* token string */

  while (token_string[i] != '\0') {
    tmp_string[i] = toupper(token_string[i]);
    i++;
  }

  tmp_string[i] = '\0';

  if (strcmp(tmp_string, "IF") == 0)
    return TRUE;
  else if (strcmp(tmp_string, "THEN") == 0)
    return TRUE;
  else if (strcmp(tmp_string, "ELSE") == 0)
    return TRUE;
  else return FALSE;
}

/*--------------------------------------------------------------*/
/*  get_word            Extract a word token and set token to   */
/*                      IDENTIFIER.                             */
/*--------------------------------------------------------------*/
void get_word()
{
    /*
    --  Extract the word.
    */
    while ((char_code(ch) == LETTER) || (char_code(ch) == DIGIT)) {
	*tokenp++ = ch;
	get_char();
    }

    *tokenp = '\0';

    if (is_reserved_word() == TRUE)
      token = RESERVED;
    else
      token = WORD;
}

/*--------------------------------------------------------------*/
/*  get_number		Extract a number token and set literal	*/
/*			to its value.  Set token to NUMBER.	*/
/*--------------------------------------------------------------*/
void get_number()
{
    int     nvalue      = 0;      /* value of number */
    int     digit_count = 0;      /* total no. of digits in number */
    BOOLEAN count_error = FALSE;  /* too many digits in number? */

    do {
	*tokenp++ = ch;

	if (++digit_count <= MAX_DIGIT_COUNT)
	    nvalue = 10*nvalue + (ch - '0');
	else count_error = TRUE;

	get_char();
    } while (char_code(ch) == DIGIT);

    if (count_error || ch != ' ') {
	token = ERROR;
	*tokenp = '\0';
	return;
    }

    literal.type          = INTEGER_LIT;
    literal.value.integer = nvalue;
    *tokenp = '\0';
    token   = NUMBER;
}

/*--------------------------------------------------------------*/
/*  get_special         Extract a special token.  The only      */
/*                      special token we recognize so far is    */
/*                      PERIOD.  All others are ERRORs.         */
/*--------------------------------------------------------------*/
void get_special()
{
    *tokenp++ = ch;
    token = (ch == '.') ? PERIOD : ERROR;
    get_char();
    *tokenp = '\0';
}

		/********************************/
		/*                              */
		/*      Source file routines	*/
		/*                              */
		/********************************/

/*--------------------------------------------------------------*/
/*  open_source_file	Open the source file and fetch its	*/
/*			first character.			*/
/*                                                              */
/*  Parameters:         name - name of source file              */
/*--------------------------------------------------------------*/
void open_source_file(char *name)
{
    if ((name == NULL) ||
	((source_file = fopen(name, "r")) == NULL)) {
	printf("*** Error:  Failed to open source file.\n");
	exit(-1);
    }

    /*
    --  Fetch the first character.
    */
    bufferp = ""        ;
    get_char();
}

/*--------------------------------------------------------------*/
/*  close_source_file	Close the source file.			*/
/*--------------------------------------------------------------*/
void close_source_file()
{
    fclose(source_file);
}

/*--------------------------------------------------------------*/
/*  get_source_line	Read the next line from the source	*/
/*                      file.  If there is one, print it out    */
/*                      and return TRUE.  Else return FALSE     */
/*                      for the end of file.                    */
/*--------------------------------------------------------------*/

BOOLEAN get_source_line()
{
    char print_buffer[MAX_SOURCE_LINE_LENGTH + 9];

    if ((fgets(source_buffer, MAX_SOURCE_LINE_LENGTH,
				    source_file)) != NULL) {
	++line_number;

	sprintf(print_buffer, "%4d: %s",
			      line_number, source_buffer);
	print_line(print_buffer);

	return(TRUE);
    }
    else return(FALSE);
}

		/********************************/
		/*                              */
		/*      Printout routines       */
		/*                              */
		/********************************/

/*--------------------------------------------------------------*/
/*  print_line          Print out a line.  Start a new page if  */
/*                      the current page is full.               */
/*                                                              */
/*  Parameters:         line - line to be printed               */
/*--------------------------------------------------------------*/

void print_line(char line[])
{
  char *save_chp = NULL;
  char tmp_buffer[MAX_SOURCE_LINE_LENGTH];

  if (++line_count > MAX_LINES_PER_PAGE) {
    print_page_header();
    line_count = 1;
  }

  /* Modified */
  if (strlen(line) > MAX_PRINT_LINE_LENGTH) {   
    do {
      save_chp = &line[MAX_PRINT_LINE_LENGTH];            
      strcpy(tmp_buffer, save_chp); 

      *save_chp = '\0';
      printf("%s\n", line);
    out_number++;

      if ((out_number + line_count) > MAX_LINES_PER_PAGE) {
	print_page_header();
	line_count = 1;
      }       

      sprintf(line, "       %s", tmp_buffer);

      if (strlen(tmp_buffer) < MAX_PRINT_LINE_LENGTH) 
	printf("       %s", tmp_buffer);

    } while (strlen(tmp_buffer) > MAX_PRINT_LINE_LENGTH);
  }
  
  if (!save_chp) 
    printf("%s", line);
}

/*--------------------------------------------------------------*/
/*  init_page_header    Initialize the fields of the page       */
/*                      header.                                 */
/*                                                              */
/*  Parameters:         name - name of source file              */
/*--------------------------------------------------------------*/
void init_page_header(char *name)
{
    time_t timer;

    strncpy(source_name, name, MAX_FILE_NAME_LENGTH - 1);

    /*
    --  Set the current date and time in the date string.
    */
    time(&timer);
    strcpy(date, (void *)asctime(localtime(&timer)));
}

/*--------------------------------------------------------------*/
/*  print_page_header   Print the page header at the top of     */
/*                      the next page.                          */
/*--------------------------------------------------------------*/
void print_page_header()
{
    putchar(FORM_FEED_CHAR);
    printf("Page %d   %s   %s\n\n", ++page_number, source_name, date);
}