LispBM
tokpar.h
Go to the documentation of this file.
1 /*
2  Copyright 2019, 2022 Joel Svensson svenssonjoel@yahoo.se
3 
4  This program is free software: you can redistribute it and/or modify
5  it under the terms of the GNU General Public License as published by
6  the Free Software Foundation, either version 3 of the License, or
7  (at your option) any later version.
8 
9  This program is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY; without even the implied warranty of
11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  GNU General Public License for more details.
13 
14  You should have received a copy of the GNU General Public License
15  along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
19 #ifndef TOKPAR_H_
20 #define TOKPAR_H_
21 
22 #include "lbm_types.h"
23 #include "lbm_channel.h"
24 
25 #define TOKOPENPAR 1u // "("
26 #define TOKCLOSEPAR 2u // ")"
27 #define TOKOPENBRACK 3u // "["
28 #define TOKCLOSEBRACK 4u // "]"
29 #define TOKDOT 5u // "."
30 #define TOKDONTCARE 6u // "_"
31 #define TOKQUOTE 7u // "'"
32 #define TOKBACKQUOTE 8u // "`"
33 #define TOKCOMMAAT 9u // ",@"
34 #define TOKCOMMA 10u // ","
35 #define TOKMATCHANY 11u // "?"
36 #define TOKOPENCURL 12u // "{"
37 #define TOKCLOSECURL 13u // "}"
38 #define TOKCONSTSTART 14u // "@const-start"
39 #define TOKCONSTEND 15u // "@const-end"
40 #define TOKCONSTSYMSTR 16u // "@const-symbol-strings"
41 
42 #define TOKTYPEBYTE 100u
43 #define TOKTYPEI 101u
44 #define TOKTYPEU 102u
45 #define TOKTYPEI32 103u
46 #define TOKTYPEU32 104u
47 #define TOKTYPEI64 105u
48 #define TOKTYPEU64 106u
49 #define TOKTYPEF32 107u
50 #define TOKTYPEF64 108u
51 
52 
53 #define TOKENIZER_ERROR 1024u
54 #define TOKENIZER_END 2048u
55 
56 // Tokenizer return values
57 // > 0 : Successfully found token
58 // = 0 : Tokenizer can definitely not create a token
59 // = -1 : Tokenizer does not know if it can or cannot create a token yet.
60 // = -2 : Tokenizer was reading a string but ran out of space (for example).
61 // This is an error!
62 // = -4 : Tokenizer was reading a symbol but ran out of space.
63 
64 #define TOKENIZER_NO_TOKEN 0
65 #define TOKENIZER_NEED_MORE -1
66 #define TOKENIZER_STRING_ERROR -2
67 #define TOKENIZER_CHAR_ERROR -3
68 #define TOKENIZER_SYMBOL_ERROR -4
69 
70 #define TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH 256
71 
72 // This is shared state between all ongoing read tasks. Maybe risky?
73 // Need to take care when dealing with this array in the reader.
75 
76 #ifdef __cplusplus
77 extern "C" {
78 #endif
79 #if 0
80 }
81 #endif
82 
85 typedef struct {
86  const char *str;
87  unsigned int pos;
88  unsigned int row;
89  unsigned int column;
91 
92 typedef struct {
93  uint32_t type;
94  uint64_t value;
95  bool negative;
96 } token_int;
97 
98 typedef struct token_float {
99  uint32_t type;
100  double value;
101  bool negative;
102 } token_float;
103 
110 int tok_syntax(lbm_char_channel_t *chan, uint32_t *res);
116 int tok_symbol(lbm_char_channel_t *chan);
122 int tok_string(lbm_char_channel_t *chan, unsigned int *string_len);
128 int tok_char(lbm_char_channel_t *chan, char *res);
134 int tok_double(lbm_char_channel_t *chan, token_float *result);
140 int tok_integer(lbm_char_channel_t *chan, token_int *result);
145 
146 #ifdef __cplusplus
147 }
148 #endif
149 #endif
token_int::value
uint64_t value
Definition: tokpar.h:94
tok_string
int tok_string(lbm_char_channel_t *chan, unsigned int *string_len)
Definition: tokpar.c:212
token_int::type
uint32_t type
Definition: tokpar.h:93
token_float
Definition: tokpar.h:98
token_int::negative
bool negative
Definition: tokpar.h:95
TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH
#define TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH
Definition: tokpar.h:70
lbm_channel.h
token_float::type
uint32_t type
Definition: tokpar.h:99
lbm_tokenizer_string_state_t::column
unsigned int column
Definition: tokpar.h:89
lbm_tokenizer_string_state_t::row
unsigned int row
Definition: tokpar.h:88
tok_symbol
int tok_symbol(lbm_char_channel_t *chan)
Definition: tokpar.c:169
lbm_char_channel_s
Definition: lbm_channel.h:68
tok_integer
int tok_integer(lbm_char_channel_t *chan, token_int *result)
Definition: tokpar.c:438
token_float::value
double value
Definition: tokpar.h:100
token_float
struct token_float token_float
tok_double
int tok_double(lbm_char_channel_t *chan, token_float *result)
Definition: tokpar.c:295
tok_char
int tok_char(lbm_char_channel_t *chan, char *res)
Definition: tokpar.c:252
token_int
Definition: tokpar.h:92
tok_syntax
int tok_syntax(lbm_char_channel_t *chan, uint32_t *res)
Definition: tokpar.c:132
tok_clean_whitespace
bool tok_clean_whitespace(lbm_char_channel_t *chan)
Definition: tokpar.c:389
lbm_types.h
lbm_tokenizer_string_state_t::str
const char * str
Definition: tokpar.h:86
token_float::negative
bool negative
Definition: tokpar.h:101
tokpar_sym_str
char tokpar_sym_str[TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH]
Definition: tokpar.c:31
lbm_tokenizer_string_state_t::pos
unsigned int pos
Definition: tokpar.h:87
lbm_tokenizer_string_state_t
Definition: tokpar.h:85