Head

GCC Code Coverage Report

Directory:	../src/		Exec	Total	Coverage
File:	/home/joels/Current/lispbm/src/tokpar.c	Lines:	245	270	90.7 %
Date:	2025-04-14 11:29:35	Branches:	207	283	73.1 %


/*
    Copyright 2019, 2021, 2022 Joel Svensson  svenssonjoel@yahoo.se

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <stdbool.h>
#include <ctype.h>
#include <string.h>
#include <stdlib.h>

#include "lbm_memory.h"
#include "lbm_types.h"
#include "lbm_channel.h"
#include "tokpar.h"
#include "symrepr.h"
#include "heap.h"
#include "env.h"

// +1 to ensure there is always a zero at last ix
char tokpar_sym_str[TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1];

typedef struct {
  const char *str;
  uint16_t token;
  uint16_t len;
} matcher;

/*
  \#\a -> 7                 ; control-g
  \#\b -> 8                 ; backspace, BS
  \#\t -> 9                 ; tab, TAB
  \#\n -> 10                ; newline
  \#\v -> 11                ; vertical tab
  \#\f -> 12                ; formfeed character
  \#\r -> 13                ; carriage return, RET
  \#\e -> 27                ; escape character, ESC
  \#\s -> 32                ; space character, SPC
  \#\\ -> 92                ; backslash character, \
  \#\d -> 127               ; delete character, DEL
*/

#define NUM_SPECIAL_CHARS 11
const char special_chars[NUM_SPECIAL_CHARS][2] =
  {{'a', '\a'},
   {'b', '\b'},
   {'t', '\t'},
   {'n', '\n'},
   {'v', '\v'},
   {'f', '\f'},
   {'r', '\r'},
   {'e', 27},
   {'s', 32},
   {'\\', '\\'},
   {'d', 127}};

#define NUM_FIXED_SIZE_TOKENS 18
const matcher fixed_size_tokens[NUM_FIXED_SIZE_TOKENS] = {
  {"(", TOKOPENPAR, 1},
  {")", TOKCLOSEPAR, 1},
  {"[|", TOKOPENARRAY, 2},
  {"[", TOKOPENBRACK, 1},
  {"]", TOKCLOSEBRACK, 1},
  {".", TOKDOT, 1},
  {"_", TOKDONTCARE, 1},
  {"'", TOKQUOTE, 1},
  {"`", TOKBACKQUOTE, 1},
  {",@", TOKCOMMAAT, 2},
  {",", TOKCOMMA, 1},
  {"?", TOKMATCHANY, 1},
  {"{", TOKOPENCURL, 1},
  {"}", TOKCLOSECURL, 1},
  {"|]", TOKCLOSEARRAY, 2},
  {"@const-start", TOKCONSTSTART, 12},
  {"@const-end", TOKCONSTEND, 10},
};

#define NUM_TYPE_QUALIFIERS 9
const matcher type_qual_table[NUM_TYPE_QUALIFIERS] = {
  {"f64", TOKTYPEF64,  3},
  {"f32", TOKTYPEF32,  3},
  {"i64", TOKTYPEI64,  3},
  {"u64", TOKTYPEU64,  3},
  {"i32", TOKTYPEI32,  3},
  {"u32", TOKTYPEU32,  3},
  {"i"  , TOKTYPEI,    1},
  {"u"  , TOKTYPEU,    1},
  {"b"  , TOKTYPEBYTE, 1}
};

static int tok_match_fixed_size_tokens(lbm_char_channel_t *chan, const matcher *m, unsigned int start_pos, unsigned int num, uint32_t *res) {

  for (unsigned int i = 0; i < num; i ++) {
    uint32_t tok_len = m[i].len;
    const char *match_str = m[i].str;
    char c;
    int char_pos;
    for (char_pos = 0; char_pos < (int)tok_len; char_pos ++) {
      int r = lbm_channel_peek(chan,(unsigned int)char_pos + start_pos, &c);
      if (r == CHANNEL_SUCCESS) {
        if (c != match_str[char_pos]) break;
      } else if (r == CHANNEL_MORE ) {
        return TOKENIZER_NEED_MORE;
      } else {
        break;
      }
    }

    if (char_pos == (int)tok_len) { //match
      *res = m[i].token;
      return (int)tok_len;
    }
  }
  return TOKENIZER_NO_TOKEN;
}

int tok_syntax(lbm_char_channel_t *chan, uint32_t *res) {
  return tok_match_fixed_size_tokens(chan, fixed_size_tokens, 0, NUM_FIXED_SIZE_TOKENS, res);
}

static bool alpha_char(char c) {
  return ((c >= 'a' && c <= 'z') ||
          (c >= 'A' && c <= 'Z'));
}

static bool num_char(char c) {
  return (c >= '0' && c <= '9');
}

static bool symchar0(char c) {
  const char *allowed = "+-*/=<>#!";

  if (alpha_char(c)) return true;
  int i = 0;
  while (allowed[i] != 0) {
    if (c == allowed[i]) return true;
    i ++;
  }
  return false;
}

static bool symchar(char c) {
  const char *allowed = "+-*/=<>!?_";

  if (alpha_char(c) || num_char(c)) return true;
  int i = 0;
  while (allowed[i] != 0) {
    if (c == allowed[i]) return true;
    i++;
  }
  return false;
}

int tok_symbol(lbm_char_channel_t *chan) {

  char c;
  int r = 0;

  r = lbm_channel_peek(chan, 0, &c);
  if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  if (r == CHANNEL_END)  return TOKENIZER_NO_TOKEN;
  if (r == CHANNEL_SUCCESS && !symchar0(c)) {
    return TOKENIZER_NO_TOKEN;
  }
  memset(tokpar_sym_str,0,TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1);
  tokpar_sym_str[0] = (char)tolower(c);

  int len = 1;

  r = lbm_channel_peek(chan,(unsigned int)len, &c);
  while (r == CHANNEL_SUCCESS && symchar(c)) {
    if (len >= 255) return TOKENIZER_SYMBOL_ERROR;
    c = (char)tolower(c);
    if (len < TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH) {
      tokpar_sym_str[len] = (char)c;
    }
    len ++;
    r = lbm_channel_peek(chan,(unsigned int)len, &c);
  }
  if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  tokpar_sym_str[len] = 0;
  return len;
}

static char translate_escape_char(char c) {
  switch(c) {
  case '\\': return '\\';
  case 'n': return '\n';
  case 'r': return '\r';
  case 't': return '\t';
  case '0': return '\0';
  case '\"': return '\"';
  default: return '\\';
  }
}

int tok_string(lbm_char_channel_t *chan, unsigned int *string_len) {

  unsigned int n = 0;
  unsigned int len = 0;
  char c;
  int r = 0;
  bool encode = false;

  r = lbm_channel_peek(chan,0,&c);
  if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  else if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;

  if (c != '\"') return TOKENIZER_NO_TOKEN;;
  n++;

  memset(tokpar_sym_str,0,TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1);

  // read string into buffer
  r = lbm_channel_peek(chan,n,&c);
  while (r == CHANNEL_SUCCESS && (c != '\"' || encode) &&
	 len < TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH) {
    if (c == '\\' && !encode) {
      encode = true;
    } else {
      tokpar_sym_str[len] = encode ? translate_escape_char(c) : c ;
      len++;
      encode = false;
    }
    n ++;
    r = lbm_channel_peek(chan, n, &c);
  }

  if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  if (c != '\"') return TOKENIZER_STRING_ERROR;

  *string_len = len;
  n ++;
  return (int)n;
}

int tok_char(lbm_char_channel_t *chan, char *res) {

  char c;
  int r;

  r = lbm_channel_peek(chan, 0, &c);
  if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  if (r == CHANNEL_END)  return TOKENIZER_NO_TOKEN;

  if (c != '\\') return TOKENIZER_NO_TOKEN;

  r = lbm_channel_peek(chan, 1, &c);
  if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  if (r == CHANNEL_END)  return TOKENIZER_NO_TOKEN;

  if (c != '#') return TOKENIZER_NO_TOKEN;

  r = lbm_channel_peek(chan, 2, &c);
  if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  if (r == CHANNEL_END)  return TOKENIZER_NO_TOKEN;

  if (c == '\\') {
    r = lbm_channel_peek(chan, 3, &c);
    if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
    if (r == CHANNEL_END)  return TOKENIZER_NO_TOKEN;

    bool ok = false;
    for (int i = 0; i < NUM_SPECIAL_CHARS; i ++) {
      if (c == special_chars[i][0]) {
        *res = special_chars[i][1];
        ok = true;
      }
    }
    if (ok) {
      return 4;
    } else {
      return TOKENIZER_CHAR_ERROR;
    }
  }
  *res = c;
  return 3;
}

int tok_double(lbm_char_channel_t *chan, token_float *result) {

  unsigned int n = 0;
  char fbuf[128];
  char c;
  bool valid_num = false;
  int res;

  memset(fbuf, 0, 128);

  result->type = TOKTYPEF32;
  result->negative = false;

  res = lbm_channel_peek(chan, 0, &c);
  if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
  if (c == '-') {
    n = 1;
    fbuf[0] = '-';
    result->negative = true;
  }

  res = lbm_channel_peek(chan, n, &c);
  if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
  while (c >= '0' && c <= '9') {
    fbuf[n] = c;
    n++;
    res = lbm_channel_peek(chan, n, &c);
    if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
    if (res == CHANNEL_END) break;
  }

  if (c == '.') {
    fbuf[n] = c;
    n ++;
  }

  else return TOKENIZER_NO_TOKEN;

  res = lbm_channel_peek(chan,n, &c);
  if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
  else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
  if (!(c >= '0' && c <= '9')) return TOKENIZER_NO_TOKEN;

  while (c >= '0' && c <= '9') {
    fbuf[n] = c;
    n++;
    res = lbm_channel_peek(chan, n, &c);
    if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
    if (res == CHANNEL_END) break;
  }

  if (c == 'e') {
    fbuf[n] = c;
    n++;
    res = lbm_channel_peek(chan,n, &c);
    if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
    else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
    if (!((c >= '0' && c <= '9') || c == '-')) return TOKENIZER_NO_TOKEN;

    while ((c >= '0' && c <= '9') || c == '-') {
      fbuf[n] = c;
      n++;
      res = lbm_channel_peek(chan, n, &c);
      if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
      if (res == CHANNEL_END) break;
    }
  }

  uint32_t tok_res;
  int type_len = tok_match_fixed_size_tokens(chan, type_qual_table, n, NUM_TYPE_QUALIFIERS, &tok_res);

  if (type_len == TOKENIZER_NEED_MORE) return type_len;
  if (type_len == TOKENIZER_NO_TOKEN) {
    result->type = TOKTYPEF32;
  } else {
    result->type = tok_res;
  }

  if ((result->negative && n > 1) ||
      (!result->negative && n > 0)) valid_num = true;

  if (n > 127) {
    return TOKENIZER_NO_TOKEN;
  }

  if(valid_num) {
    result->value = (double)strtod(fbuf,NULL);
    return (int)n + type_len;
  }
  return TOKENIZER_NO_TOKEN;
}

bool tok_clean_whitespace(lbm_char_channel_t *chan) {

  bool cleaning_whitespace = true;
  char c;
  int r;

  while (cleaning_whitespace) {

    if (lbm_channel_comment(chan)) {
      while (true) {
        r = lbm_channel_peek(chan, 0, &c);
        if (r == CHANNEL_END) {
          lbm_channel_set_comment(chan, false);
          cleaning_whitespace = false;
          break;
        }
        if (r == CHANNEL_MORE) {
          return false;
        }
        lbm_channel_drop(chan,1);
        if (c == '\n') {
          lbm_channel_set_comment(chan, false);
          break;
        }
      }
    }

    do {
      r = lbm_channel_peek(chan, 0, &c);
      if (r == CHANNEL_MORE) {
        return false;
      } else if (r == CHANNEL_END) {
        return true;
      }
      if (c == ';') {
        lbm_channel_set_comment(chan, true);
        break;
      }
      if (isspace(c)) {
        lbm_channel_drop(chan,1);
      } else {
        cleaning_whitespace = false;
      }

    } while (cleaning_whitespace);
  }
  return true;
}

int tok_integer(lbm_char_channel_t *chan, token_int *result) {
  uint64_t acc = 0;
  unsigned int n = 0;
  bool valid_num = false;
  char c;
  int res;

  result->type = TOKTYPEI;
  result-> negative = false;
  res = lbm_channel_peek(chan, 0, &c);
  if (res == CHANNEL_MORE) {
    return TOKENIZER_NEED_MORE;
  } else if (res == CHANNEL_END) {
    return TOKENIZER_NO_TOKEN;
  }
  if (c == '-') {
    n = 1;
    result->negative = true;
  }

  bool hex = false;
  res = lbm_channel_peek(chan, n, &c);
  if (res == CHANNEL_SUCCESS && c == '0') {
    res = lbm_channel_peek(chan, n + 1, &c);
    if ( res == CHANNEL_SUCCESS && (c == 'x' || c == 'X')) {
      hex = true;
    } else if (res == CHANNEL_MORE) {
      return TOKENIZER_NEED_MORE;
    }
  } else if (res == CHANNEL_MORE) {
    return TOKENIZER_NEED_MORE;
  }

  if (hex) {
    n += 2;

    res = lbm_channel_peek(chan,n, &c);

    if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
    else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;

    while ((c >= '0' && c <= '9') ||
           (c >= 'a' && c <= 'f') ||
           (c >= 'A' && c <= 'F')) {
      uint32_t val; /* values between 0 and 16 */
      if (c >= 'a' && c <= 'f') {
        val = 10 + (uint32_t)c - 'a';
      } else if (c >= 'A' && c <= 'F') {
        val = 10 + (uint32_t)(c - 'A');
      } else {
        val = (uint32_t)c - '0';
      }
      acc = (acc * 0x10) + val;
      n++;
      res = lbm_channel_peek(chan, n, &c);
      if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
      if (res == CHANNEL_END) break;

    }
  } else {
    res = lbm_channel_peek(chan, n, &c);
    if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
    while (c >= '0' && c <= '9') {
      acc = (acc*10) + (uint32_t)(c - '0');
      n++;
      res = lbm_channel_peek(chan, n, &c);
      if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
      if (res == CHANNEL_END)  break;
    }
  }

  if (n == 0) return TOKENIZER_NO_TOKEN;

  uint32_t tok_res;
  int type_len = tok_match_fixed_size_tokens(chan, type_qual_table, n, NUM_TYPE_QUALIFIERS, &tok_res);

  if (type_len == TOKENIZER_NEED_MORE) return type_len;
  if (type_len != TOKENIZER_NO_TOKEN) {
    result->type = tok_res;
  }

  if ((result->negative && n > 1) ||
      !result->negative) valid_num = true;

  if (valid_num) {
    result->value = acc;
    return (int)n + type_len;
  }
  return TOKENIZER_NO_TOKEN;
}


Generated by: GCOVR (Version 4.2)

Line	Branch	Exec	Source
1			/*
2			Copyright 2019, 2021, 2022 Joel Svensson svenssonjoel@yahoo.se
3
4			This program is free software: you can redistribute it and/or modify
5			it under the terms of the GNU General Public License as published by
6			the Free Software Foundation, either version 3 of the License, or
7			(at your option) any later version.
8
9			This program is distributed in the hope that it will be useful,
10			but WITHOUT ANY WARRANTY; without even the implied warranty of
11			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12			GNU General Public License for more details.
13
14			You should have received a copy of the GNU General Public License
15			along with this program. If not, see <http://www.gnu.org/licenses/>.
16			*/
17
18			#include <stdbool.h>
19			#include <ctype.h>
20			#include <string.h>
21			#include <stdlib.h>
22
23			#include "lbm_memory.h"
24			#include "lbm_types.h"
25			#include "lbm_channel.h"
26			#include "tokpar.h"
27			#include "symrepr.h"
28			#include "heap.h"
29			#include "env.h"
30
31			// +1 to ensure there is always a zero at last ix
32			char tokpar_sym_str[TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1];
33
34			typedef struct {
35			const char *str;
36			uint16_t token;
37			uint16_t len;
38			} matcher;
39
40			/*
41			\#\a -> 7 ; control-g
42			\#\b -> 8 ; backspace, BS
43			\#\t -> 9 ; tab, TAB
44			\#\n -> 10 ; newline
45			\#\v -> 11 ; vertical tab
46			\#\f -> 12 ; formfeed character
47			\#\r -> 13 ; carriage return, RET
48			\#\e -> 27 ; escape character, ESC
49			\#\s -> 32 ; space character, SPC
50			\#\\ -> 92 ; backslash character, \
51			\#\d -> 127 ; delete character, DEL
52			*/
53
54			#define NUM_SPECIAL_CHARS 11
55			const char special_chars[NUM_SPECIAL_CHARS][2] =
56			{{'a', '\a'},
57			{'b', '\b'},
58			{'t', '\t'},
59			{'n', '\n'},
60			{'v', '\v'},
61			{'f', '\f'},
62			{'r', '\r'},
63			{'e', 27},
64			{'s', 32},
65			{'\\', '\\'},
66			{'d', 127}};
67
68			#define NUM_FIXED_SIZE_TOKENS 18
69			const matcher fixed_size_tokens[NUM_FIXED_SIZE_TOKENS] = {
70			{"(", TOKOPENPAR, 1},
71			{")", TOKCLOSEPAR, 1},
72			{"[\|", TOKOPENARRAY, 2},
73			{"[", TOKOPENBRACK, 1},
74			{"]", TOKCLOSEBRACK, 1},
75			{".", TOKDOT, 1},
76			{"_", TOKDONTCARE, 1},
77			{"'", TOKQUOTE, 1},
78			{"`", TOKBACKQUOTE, 1},
79			{",@", TOKCOMMAAT, 2},
80			{",", TOKCOMMA, 1},
81			{"?", TOKMATCHANY, 1},
82			{"{", TOKOPENCURL, 1},
83			{"}", TOKCLOSECURL, 1},
84			{"\|]", TOKCLOSEARRAY, 2},
85			{"@const-start", TOKCONSTSTART, 12},
86			{"@const-end", TOKCONSTEND, 10},
87			};
88
89			#define NUM_TYPE_QUALIFIERS 9
90			const matcher type_qual_table[NUM_TYPE_QUALIFIERS] = {
91			{"f64", TOKTYPEF64, 3},
92			{"f32", TOKTYPEF32, 3},
93			{"i64", TOKTYPEI64, 3},
94			{"u64", TOKTYPEU64, 3},
95			{"i32", TOKTYPEI32, 3},
96			{"u32", TOKTYPEU32, 3},
97			{"i" , TOKTYPEI, 1},
98			{"u" , TOKTYPEU, 1},
99			{"b" , TOKTYPEBYTE, 1}
100			};
101
102		9144424	static int tok_match_fixed_size_tokens(lbm_char_channel_t chan, const matcher m, unsigned int start_pos, unsigned int num, uint32_t *res) {
103
104	✓✓	113969778	for (unsigned int i = 0; i < num; i ++) {
105		110618850	uint32_t tok_len = m[i].len;
106		110618850	const char *match_str = m[i].str;
107			char c;
108			int char_pos;
109	✓✓	112184952	for (char_pos = 0; char_pos < (int)tok_len; char_pos ++) {
110		106391456	int r = lbm_channel_peek(chan,(unsigned int)char_pos + start_pos, &c);
111	✓✓	106391456	if (r == CHANNEL_SUCCESS) {
112	✓✓	106390896	if (c != match_str[char_pos]) break;
113	✗✓	560	} else if (r == CHANNEL_MORE ) {
114		5793496	return TOKENIZER_NEED_MORE;
115			} else {
116		560	break;
117			}
118			}
119
120	✓✓	110618850	if (char_pos == (int)tok_len) { //match
121		5793496	*res = m[i].token;
122		5793496	return (int)tok_len;
123			}
124			}
125		3350928	return TOKENIZER_NO_TOKEN;
126			}
127
128		5766280	int tok_syntax(lbm_char_channel_t chan, uint32_t res) {
129		5766280	return tok_match_fixed_size_tokens(chan, fixed_size_tokens, 0, NUM_FIXED_SIZE_TOKENS, res);
130			}
131
132		3626426	static bool alpha_char(char c) {
133	✓✓✓✓ ✓✓	3633762	return ((c >= 'a' && c <= 'z') \|\|
134	✓✓	7336	(c >= 'A' && c <= 'Z'));
135			}
136
137		1112158	static bool num_char(char c) {
138	✓✓✓✓	1112158	return (c >= '0' && c <= '9');
139			}
140
141		924734	static bool symchar0(char c) {
142		924734	const char allowed = "+-/=<>#!";
143
144	✓✓	924734	if (alpha_char(c)) return true;
145		57596	int i = 0;
146	✓✓	182532	while (allowed[i] != 0) {
147	✓✓	182364	if (c == allowed[i]) return true;
148		124936	i ++;
149			}
150		168	return false;
151			}
152
153		2701692	static bool symchar(char c) {
154		2701692	const char allowed = "+-/=<>!?_";
155
156	✓✓✓✓	2701692	if (alpha_char(c) \|\| num_char(c)) return true;
157		994611	int i = 0;
158	✓✓	10260414	while (allowed[i] != 0) {
159	✓✓	9347534	if (c == allowed[i]) return true;
160		9265803	i++;
161			}
162		912880	return false;
163			}
164
165		924734	int tok_symbol(lbm_char_channel_t *chan) {
166
167			char c;
168		924734	int r = 0;
169
170		924734	r = lbm_channel_peek(chan, 0, &c);
171	✗✓	924734	if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
172	✗✓	924734	if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
173	✓✗✓✓	924734	if (r == CHANNEL_SUCCESS && !symchar0(c)) {
174		168	return TOKENIZER_NO_TOKEN;
175			}
176		924566	memset(tokpar_sym_str,0,TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1);
177		924566	tokpar_sym_str[0] = (char)tolower(c);
178
179		924566	int len = 1;
180
181		924566	r = lbm_channel_peek(chan,(unsigned int)len, &c);
182	✓✓✓✓	2713378	while (r == CHANNEL_SUCCESS && symchar(c)) {
183	✗✓	1788812	if (len >= 255) return TOKENIZER_SYMBOL_ERROR;
184		1788812	c = (char)tolower(c);
185	✓✗	1788812	if (len < TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH) {
186		1788812	tokpar_sym_str[len] = (char)c;
187			}
188		1788812	len ++;
189		1788812	r = lbm_channel_peek(chan,(unsigned int)len, &c);
190			}
191	✓✓	924566	if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
192		924556	tokpar_sym_str[len] = 0;
193		924556	return len;
194			}
195
196		224	static char translate_escape_char(char c) {
197	✗✓✗✗ ✗✓✗	224	switch(c) {
198			case '\\': return '\\';
199		56	case 'n': return '\n';
200			case 'r': return '\r';
201			case 't': return '\t';
202			case '0': return '\0';
203		168	case '\"': return '\"';
204			default: return '\\';
205			}
206			}
207
208		4308678	int tok_string(lbm_char_channel_t chan, unsigned int string_len) {
209
210		4308678	unsigned int n = 0;
211		4308678	unsigned int len = 0;
212			char c;
213		4308678	int r = 0;
214		4308678	bool encode = false;
215
216		4308678	r = lbm_channel_peek(chan,0,&c);
217	✗✓	4308678	if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
218	✗✓	4308678	else if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
219
220	✓✓	4308678	if (c != '\"') return TOKENIZER_NO_TOKEN;;
221		9381	n++;
222
223		9381	memset(tokpar_sym_str,0,TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1);
224
225			// read string into buffer
226		9381	r = lbm_channel_peek(chan,n,&c);
227	✓✓✓✓ ✓✓✓✗	61938	while (r == CHANNEL_SUCCESS && (c != '\"' \|\| encode) &&
228			len < TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH) {
229	✓✓✓✗	52557	if (c == '\\' && !encode) {
230		224	encode = true;
231			} else {
232	✓✓	52333	tokpar_sym_str[len] = encode ? translate_escape_char(c) : c ;
233		52333	len++;
234		52333	encode = false;
235			}
236		52557	n ++;
237		52557	r = lbm_channel_peek(chan, n, &c);
238			}
239
240	✓✓	9381	if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
241	✗✓	9380	if (c != '\"') return TOKENIZER_STRING_ERROR;
242
243		9380	*string_len = len;
244		9380	n ++;
245		9380	return (int)n;
246			}
247
248		168	int tok_char(lbm_char_channel_t chan, char res) {
249
250			char c;
251			int r;
252
253		168	r = lbm_channel_peek(chan, 0, &c);
254	✗✓	168	if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
255	✗✓	168	if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
256
257	✗✓	168	if (c != '\\') return TOKENIZER_NO_TOKEN;
258
259		168	r = lbm_channel_peek(chan, 1, &c);
260	✗✓	168	if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
261	✗✓	168	if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
262
263	✗✓	168	if (c != '#') return TOKENIZER_NO_TOKEN;
264
265		168	r = lbm_channel_peek(chan, 2, &c);
266	✗✓	168	if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
267	✗✓	168	if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
268
269	✗✓	168	if (c == '\\') {
270			r = lbm_channel_peek(chan, 3, &c);
271			if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
272			if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
273
274			bool ok = false;
275			for (int i = 0; i < NUM_SPECIAL_CHARS; i ++) {
276			if (c == special_chars[i][0]) {
277			*res = special_chars[i][1];
278			ok = true;
279			}
280			}
281			if (ok) {
282			return 4;
283			} else {
284			return TOKENIZER_CHAR_ERROR;
285			}
286			}
287		168	*res = c;
288		168	return 3;
289			}
290
291		4299297	int tok_double(lbm_char_channel_t chan, token_float result) {
292
293		4299297	unsigned int n = 0;
294			char fbuf[128];
295			char c;
296		4299297	bool valid_num = false;
297			int res;
298
299		4299297	memset(fbuf, 0, 128);
300
301		4299297	result->type = TOKTYPEF32;
302		4299297	result->negative = false;
303
304		4299297	res = lbm_channel_peek(chan, 0, &c);
305	✗✓	4299297	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
306	✗✓	4299297	else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
307	✓✓	4299297	if (c == '-') {
308		5600	n = 1;
309		5600	fbuf[0] = '-';
310		5600	result->negative = true;
311			}
312
313		4299297	res = lbm_channel_peek(chan, n, &c);
314	✗✓	4299297	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
315	✗✓	4299297	else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
316	✓✓✓✓	8566500	while (c >= '0' && c <= '9') {
317		4267261	fbuf[n] = c;
318		4267261	n++;
319		4267261	res = lbm_channel_peek(chan, n, &c);
320	✓✓	4267261	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
321	✓✓	4267259	if (res == CHANNEL_END) break;
322			}
323
324	✓✓	4299295	if (c == '.') {
325		11928	fbuf[n] = c;
326		11928	n ++;
327			}
328
329		4287367	else return TOKENIZER_NO_TOKEN;
330
331		11928	res = lbm_channel_peek(chan,n, &c);
332	✗✓	11928	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
333	✗✓	11928	else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
334	✓✗✗✓	11928	if (!(c >= '0' && c <= '9')) return TOKENIZER_NO_TOKEN;
335
336	✓✓✓✓	28784	while (c >= '0' && c <= '9') {
337		16856	fbuf[n] = c;
338		16856	n++;
339		16856	res = lbm_channel_peek(chan, n, &c);
340	✗✓	16856	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
341	✗✓	16856	if (res == CHANNEL_END) break;
342			}
343
344	✓✓	11928	if (c == 'e') {
345		112	fbuf[n] = c;
346		112	n++;
347		112	res = lbm_channel_peek(chan,n, &c);
348	✗✓	112	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
349	✗✓	112	else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
350	✓✗✗✓ ✗✗	112	if (!((c >= '0' && c <= '9') \|\| c == '-')) return TOKENIZER_NO_TOKEN;
351
352	✓✓✓✓ ✗✓	224	while ((c >= '0' && c <= '9') \|\| c == '-') {
353		112	fbuf[n] = c;
354		112	n++;
355		112	res = lbm_channel_peek(chan, n, &c);
356	✗✓	112	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
357	✗✓	112	if (res == CHANNEL_END) break;
358			}
359			}
360
361			uint32_t tok_res;
362		11928	int type_len = tok_match_fixed_size_tokens(chan, type_qual_table, n, NUM_TYPE_QUALIFIERS, &tok_res);
363
364	✗✓	11928	if (type_len == TOKENIZER_NEED_MORE) return type_len;
365	✓✓	11928	if (type_len == TOKENIZER_NO_TOKEN) {
366		5908	result->type = TOKTYPEF32;
367			} else {
368		6020	result->type = tok_res;
369			}
370
371	✓✓✗✓	11928	if ((result->negative && n > 1) \|\|
372	✓✗✓✗	11928	(!result->negative && n > 0)) valid_num = true;
373
374	✗✓	11928	if (n > 127) {
375			return TOKENIZER_NO_TOKEN;
376			}
377
378	✓✗	11928	if(valid_num) {
379		11928	result->value = (double)strtod(fbuf,NULL);
380		11928	return (int)n + type_len;
381			}
382			return TOKENIZER_NO_TOKEN;
383			}
384
385		5788716	bool tok_clean_whitespace(lbm_char_channel_t *chan) {
386
387		5788716	bool cleaning_whitespace = true;
388			char c;
389			int r;
390
391	✓✓	11560988	while (cleaning_whitespace) {
392
393	✓✓	5794708	if (lbm_channel_comment(chan)) {
394			while (true) {
395		683899	r = lbm_channel_peek(chan, 0, &c);
396	✗✓	683899	if (r == CHANNEL_END) {
397			lbm_channel_set_comment(chan, false);
398			cleaning_whitespace = false;
399			break;
400			}
401	✓✓	683899	if (r == CHANNEL_MORE) {
402		698	return false;
403			}
404		683201	lbm_channel_drop(chan,1);
405	✓✓	683201	if (c == '\n') {
406		5988	lbm_channel_set_comment(chan, false);
407		5988	break;
408			}
409			}
410			}
411
412			do {
413		10389940	r = lbm_channel_peek(chan, 0, &c);
414	✓✓	10389940	if (r == CHANNEL_MORE) {
415		22	return false;
416	✓✓	10389918	} else if (r == CHANNEL_END) {
417		21716	return true;
418			}
419	✓✓	10368202	if (c == ';') {
420		5992	lbm_channel_set_comment(chan, true);
421		5992	break;
422			}
423	✓✓	10362210	if (isspace(c)) {
424		4595930	lbm_channel_drop(chan,1);
425			} else {
426		5766280	cleaning_whitespace = false;
427			}
428
429	✓✓	10362210	} while (cleaning_whitespace);
430			}
431		5766280	return true;
432			}
433
434		4287367	int tok_integer(lbm_char_channel_t chan, token_int result) {
435		4287367	uint64_t acc = 0;
436		4287367	unsigned int n = 0;
437		4287367	bool valid_num = false;
438			char c;
439			int res;
440
441		4287367	result->type = TOKTYPEI;
442		4287367	result-> negative = false;
443		4287367	res = lbm_channel_peek(chan, 0, &c);
444	✗✓	4287367	if (res == CHANNEL_MORE) {
445			return TOKENIZER_NEED_MORE;
446	✗✓	4287367	} else if (res == CHANNEL_END) {
447			return TOKENIZER_NO_TOKEN;
448			}
449	✓✓	4287367	if (c == '-') {
450		5348	n = 1;
451		5348	result->negative = true;
452			}
453
454		4287367	bool hex = false;
455		4287367	res = lbm_channel_peek(chan, n, &c);
456	✓✗✓✓	4287367	if (res == CHANNEL_SUCCESS && c == '0') {
457		29317	res = lbm_channel_peek(chan, n + 1, &c);
458	✓✗✓✓ ✗✓	29317	if ( res == CHANNEL_SUCCESS && (c == 'x' \|\| c == 'X')) {
459		15737	hex = true;
460	✗✓	13580	} else if (res == CHANNEL_MORE) {
461			return TOKENIZER_NEED_MORE;
462			}
463	✗✓	4258050	} else if (res == CHANNEL_MORE) {
464			return TOKENIZER_NEED_MORE;
465			}
466
467	✓✓	4287367	if (hex) {
468		15737	n += 2;
469
470		15737	res = lbm_channel_peek(chan,n, &c);
471
472	✓✓	15737	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
473	✗✓	15736	else if (res == CHANNEL_END) return TOKENIZER_NO_TOKEN;
474
475	✓✓✓✓	51688	while ((c >= '0' && c <= '9') \|\|
476	✓✓✓✓	25592	(c >= 'a' && c <= 'f') \|\|
477	✓✓✓✓	25564	(c >= 'A' && c <= 'F')) {
478			uint32_t val; /* values between 0 and 16 */
479	✓✓✓✗	35952	if (c >= 'a' && c <= 'f') {
480		28	val = 10 + (uint32_t)c - 'a';
481	✓✓✓✗	35924	} else if (c >= 'A' && c <= 'F') {
482		9828	val = 10 + (uint32_t)(c - 'A');
483			} else {
484		26096	val = (uint32_t)c - '0';
485			}
486		35952	acc = (acc * 0x10) + val;
487		35952	n++;
488		35952	res = lbm_channel_peek(chan, n, &c);
489	✗✓	35952	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
490	✗✓	35952	if (res == CHANNEL_END) break;
491
492			}
493			} else {
494		4271630	res = lbm_channel_peek(chan, n, &c);
495	✗✓	4271630	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
496	✓✓✓✓	8510494	while (c >= '0' && c <= '9') {
497		4238920	acc = (acc*10) + (uint32_t)(c - '0');
498		4238920	n++;
499		4238920	res = lbm_channel_peek(chan, n, &c);
500	✗✓	4238920	if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
501	✓✓	4238920	if (res == CHANNEL_END) break;
502			}
503			}
504
505	✓✓	4287366	if (n == 0) return TOKENIZER_NO_TOKEN;
506
507			uint32_t tok_res;
508		3366216	int type_len = tok_match_fixed_size_tokens(chan, type_qual_table, n, NUM_TYPE_QUALIFIERS, &tok_res);
509
510	✗✓	3366216	if (type_len == TOKENIZER_NEED_MORE) return type_len;
511	✓✓	3366216	if (type_len != TOKENIZER_NO_TOKEN) {
512		21196	result->type = tok_res;
513			}
514
515	✓✓✓✓	3366216	if ((result->negative && n > 1) \|\|
516	✓✓	3366216	!result->negative) valid_num = true;
517
518	✓✓	3366216	if (valid_num) {
519		3362632	result->value = acc;
520		3362632	return (int)n + type_len;
521			}
522		3584	return TOKENIZER_NO_TOKEN;
523			}