tokpar.c File Reference

#include <stdbool.h>
#include <ctype.h>
#include <string.h>
#include <stdlib.h>
#include "lbm_memory.h"
#include "lbm_types.h"
#include "lbm_channel.h"
#include "tokpar.h"
#include "symrepr.h"
#include "heap.h"
#include "env.h"

Include dependency graph for tokpar.c:

Data Structures
struct	matcher

Macros
#define	NUM_SPECIAL_CHARS   11
#define	NUM_FIXED_SIZE_TOKENS   16
#define	NUM_TYPE_QUALIFIERS   9

Functions
static void	clear_sym_str (void)
static int	tok_match_fixed_size_tokens (lbm_char_channel_t *chan, const matcher *m, unsigned int start_pos, unsigned int num, uint32_t *res)
int	tok_syntax (lbm_char_channel_t *chan, uint32_t *res)
static bool	alpha_char (char c)
static bool	num_char (char c)
static bool	symchar0 (char c)
static bool	symchar (char c)
int	tok_symbol (lbm_char_channel_t *chan)
static char	translate_escape_char (char c)
int	tok_string (lbm_char_channel_t *chan, unsigned int *string_len)
int	tok_char (lbm_char_channel_t *chan, char *res)
int	tok_double (lbm_char_channel_t *chan, token_float *result)
bool	tok_clean_whitespace (lbm_char_channel_t *chan)
int	tok_integer (lbm_char_channel_t *chan, token_int *result)

Variables
char	tokpar_sym_str [TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH]
const char	special_chars [NUM_SPECIAL_CHARS][2]
const matcher	fixed_size_tokens [NUM_FIXED_SIZE_TOKENS]
const matcher	type_qual_table [NUM_TYPE_QUALIFIERS]

Macro Definition Documentation

NUM_FIXED_SIZE_TOKENS

#define NUM_FIXED_SIZE_TOKENS   16

NUM_SPECIAL_CHARS

#define NUM_SPECIAL_CHARS   11

NUM_TYPE_QUALIFIERS

#define NUM_TYPE_QUALIFIERS   9

Function Documentation

alpha_char()

static bool alpha_char ( char  c )

static

clear_sym_str()

static void clear_sym_str ( void  )

static

num_char()

static bool num_char ( char  c )

static

symchar()

static bool symchar ( char  c )

static

symchar0()

static bool symchar0 ( char  c )

static

tok_char()

int tok_char	(	lbm_char_channel_t *	chan,
		char *	res
	)

Attempt to read a character from the character stream.

Parameters

chan	Character channel to read characters from.
res	Result character if successful.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_clean_whitespace()

bool tok_clean_whitespace

(

lbm_char_channel_t *

chan

)

Clean off whitespace from head of the character stream

Returns

True if whitespace could be cleaned. False if stream has no more available characters at the moment.

tok_double()

int tok_double	(	lbm_char_channel_t *	chan,
		token_float *	result
	)

Attempt to read a floating point number from the character stream.

Parameters

chan	Character channel to read characters from.
res	Result token_float-object if successful.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_integer()

int tok_integer	(	lbm_char_channel_t *	chan,
		token_int *	result
	)

Attempt to read an integer value from the character stream.

Parameters

chan	Character channel to read characters from.
res	Result token_int-object if successful.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_match_fixed_size_tokens()

static int tok_match_fixed_size_tokens ( lbm_char_channel_t *  chan, const matcher *  m, unsigned int  start_pos, unsigned int  num, uint32_t *  res  )

static

tok_string()

int tok_string	(	lbm_char_channel_t *	chan,
		unsigned int *	string_len
	)

Attempt to read a string from the character stream. If successful the resulting string is found in tokpar_sym_str.

Parameters

chan	Character channel to read characters from.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_symbol()

int tok_symbol

(

lbm_char_channel_t *

chan

)

Attempt to read a symbol from the character stream. If successful the resulting symbol string is found in tokpar_sym_str.

Parameters

chan	Character channel to read characters from.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_syntax()

int tok_syntax	(	lbm_char_channel_t *	chan,
		uint32_t *	res
	)

Attempt to read a "syntax" token from the character stream. A "syntax" token is for example '(', ')', '@' and so on.

Parameters

chan	Character channel to read characters from.
res	Result token identifier.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

translate_escape_char()

static char translate_escape_char ( char  c )

static

Variable Documentation

fixed_size_tokens

const matcher fixed_size_tokens[NUM_FIXED_SIZE_TOKENS]

Initial value:

= {
  {"(", TOKOPENPAR, 1},
  {")", TOKCLOSEPAR, 1},
  {"[", TOKOPENBRACK, 1},
  {"]", TOKCLOSEBRACK, 1},
  {".", TOKDOT, 1},
  {"_", TOKDONTCARE, 1},
  {"'", TOKQUOTE, 1},
  {"`", TOKBACKQUOTE, 1},
  {",@", TOKCOMMAAT, 2},
  {",", TOKCOMMA, 1},
  {"?", TOKMATCHANY, 1},
  {"{", TOKOPENCURL, 1},
  {"}", TOKCLOSECURL, 1},
  {"@const-start", TOKCONSTSTART, 12},
  {"@const-end", TOKCONSTEND, 10},
  {"@const-symbol-strings", TOKCONSTSYMSTR, 21},
}

special_chars

const char special_chars[NUM_SPECIAL_CHARS][2]

Initial value:

=
  {{'a', '\a'},
   {'b', '\b'},
   {'t', '\t'},
   {'n', '\n'},
   {'v', '\v'},
   {'f', '\f'},
   {'r', '\r'},
   {'e', 27},
   {'s', 32},
   {'\\', '\\'},
   {'d', 127}}

tokpar_sym_str

char tokpar_sym_str[TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH]

type_qual_table

const matcher type_qual_table[NUM_TYPE_QUALIFIERS]

Initial value:

= {
  {"f64", TOKTYPEF64,  3},
  {"f32", TOKTYPEF32,  3},
  {"i64", TOKTYPEI64,  3},
  {"u64", TOKTYPEU64,  3},
  {"i32", TOKTYPEI32,  3},
  {"u32", TOKTYPEU32,  3},
  {"i"  , TOKTYPEI,    1},
  {"u"  , TOKTYPEU,    1},
  {"b"  , TOKTYPEBYTE, 1}
}