Edinburgh Speech Tools 2.4-release
 
Loading...
Searching...
No Matches
EST_Token.cc
1/*************************************************************************/
2/* */
3/* Centre for Speech Technology Research */
4/* University of Edinburgh, UK */
5/* Copyright (c) 1996 */
6/* All Rights Reserved. */
7/* */
8/* Permission is hereby granted, free of charge, to use and distribute */
9/* this software and its documentation without restriction, including */
10/* without limitation the rights to use, copy, modify, merge, publish, */
11/* distribute, sublicense, and/or sell copies of this work, and to */
12/* permit persons to whom this work is furnished to do so, subject to */
13/* the following conditions: */
14/* 1. The code must retain the above copyright notice, this list of */
15/* conditions and the following disclaimer. */
16/* 2. Any modifications must be clearly marked as such. */
17/* 3. Original authors' names are not deleted. */
18/* 4. The authors' names are not used to endorse or promote products */
19/* derived from this software without specific prior written */
20/* permission. */
21/* */
22/* THE UNIVERSITY OF EDINBURGH AND THE CONTRIBUTORS TO THIS WORK */
23/* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING */
24/* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT */
25/* SHALL THE UNIVERSITY OF EDINBURGH NOR THE CONTRIBUTORS BE LIABLE */
26/* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES */
27/* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN */
28/* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, */
29/* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF */
30/* THIS SOFTWARE. */
31/* */
32/*************************************************************************/
33/* Author : Alan W Black */
34/* Date : April 1996 */
35/*-----------------------------------------------------------------------*/
36/* */
37/* A Tokenize class, both for Tokens (Strings plus alpha) */
38/* EST_TokenStream for strings, FILE *, files, pipes etc */
39/* */
40/*=======================================================================*/
41#include <cstdio>
42#include <iostream>
43#include "EST_unix.h"
44#include <cstdlib>
45#include <climits>
46#include <cstring>
47#include "EST_math.h"
48#include "EST_Token.h"
49#include "EST_string_aux.h"
50#include "EST_cutils.h"
51#include "EST_error.h"
52
53const EST_String EST_Token_Default_WhiteSpaceChars = " \t\n\r";
54const EST_String EST_Token_Default_SingleCharSymbols = "(){}[]";
55const EST_String EST_Token_Default_PrePunctuationSymbols = "\"'`({[";
56const EST_String EST_Token_Default_PunctuationSymbols = "\"'`.,:;!?]})";
57const EST_String Token_Origin_FD = "existing file descriptor";
58const EST_String Token_Origin_Stream = "existing istream";
59const EST_String Token_Origin_String = "existing string";
60
61static EST_Regex RXanywhitespace("[ \t\n\r]");
62
63static inline char *check_extend_str_in(char *str, int pos, int *max)
64{
65 // Check we are not at the end of the string, if so get some more
66 // and copy the old one into the new one
67 char *newstuff;
68
69 if (pos >= *max)
70 {
71 if (pos > *max)
72 *max = 2 * pos;
73 else
74 *max *= 2;
75 newstuff = new char[*max];
76 strncpy(newstuff,str,pos);
77 delete [] str;
78 return newstuff;
79 }
80 else
81 return str;
82}
83
84#define check_extend_str(STR, POS, MAX) \
85 (((POS)>= *(MAX))?check_extend_str_in((STR),(POS),(MAX)):(STR))
86
88{
89 s << "[TOKEN " << p.pname << "]";
90 return s;
91}
92
93
94EST_Token &EST_Token::operator = (const EST_Token &a)
95{
96 linenum = a.linenum;
97 linepos = a.linepos;
98 p_filepos = a.p_filepos;
99 p_quoted = a.p_quoted;
100 space = a.space;
101 prepunc = a.prepunc;
102 pname = a.pname;
103 punc = a.punc;
104 return *this;
105}
106
108{
109 return "line "+itoString(linenum)+" char "+itoString(linepos);
110}
111
112EST_Token &EST_Token::operator = (const EST_String &a)
113{
114 pname = a;
115 return *this;
116}
117
118EST_TokenStream::EST_TokenStream()
119{
120 tok_wspacelen = 64; // will grow if necessary
121 tok_wspace = new char[tok_wspacelen];
122 tok_stufflen = 512; // will grow if necessary
123 tok_stuff = new char[tok_stufflen];
124 tok_prepuncslen = 32; // will grow if necessary
125 tok_prepuncs = new char[tok_prepuncslen];
126
127 default_values();
128}
129
130EST_TokenStream::EST_TokenStream(EST_TokenStream &s)
131{
132 (void)s;
133
134 cerr << "TokenStream: warning passing TokenStream not as reference"
135 << endl;
136
137 // You *really* shouldn't use this AT ALL unless you
138 // fully understand its consequences, you'll be copying open
139 // files and moving file pointers all over the place
140 // basically *DON'T* do this, pass the stream by reference
141
142 // Now there may be occasions when you do want to do this for example
143 // when you need to do far look ahead or check point as you read
144 // but they are obscure and I'm not sure how to do that for all
145 // the file forms supported by the TokenStream. If you do
146 // I can write a clone function that might do it.
147
148}
149
150void EST_TokenStream::default_values()
151{
152 type = tst_none;
153 peeked_tokp = FALSE;
154 peeked_charp = FALSE;
155 eof_flag = FALSE;
156 quotes = FALSE;
157 p_filepos = 0;
158 linepos = 1;
159 WhiteSpaceChars = EST_Token_Default_WhiteSpaceChars;
160 SingleCharSymbols = EST_String::Empty;
161 PrePunctuationSymbols = EST_String::Empty;
162 PunctuationSymbols = EST_String::Empty;
163 build_table();
164 close_at_end=TRUE;
165}
166
168{
169 if (type != tst_none)
170 close();
171 delete [] tok_wspace;
172 delete [] tok_stuff;
173 delete [] tok_prepuncs;
174
175}
176
178{
179 s << "[TOKENSTREAM ";
180 switch (p.type)
181 {
182 case tst_none:
183 cerr << "UNSET"; break;
184 case tst_file:
185 cerr << "FILE"; break;
186 case tst_pipe:
187 cerr << "PIPE"; break;
188 case tst_istream:
189 cerr << "ISTREAM"; break;
190 case tst_string:
191 cerr << "STRING"; break;
192 default:
193 cerr << "UNKNOWN" << endl;
194 }
195 s << "]";
196
197 return s;
198}
199
201{
202 if (type != tst_none)
203 close();
204 default_values();
205 fp = fopen(filename,"rb");
206 if (fp == NULL)
207 {
208 cerr << "Cannot open file " << filename << " as tokenstream"
209 << endl;
210 return -1;
211 }
212 Origin = filename;
213 type = tst_file;
214
215 return 0;
216}
217
219{
220 // absorb already open stream
221 if (type != tst_none)
222 close();
223 default_values();
224 fp = ofp;
225 if (fp == NULL)
226 {
227 cerr << "Cannot absorb NULL filestream as tokenstream" << endl;
228 return -1;
229 }
230 Origin = Token_Origin_FD;
231 type = tst_file;
232
233 close_at_end = close_when_finished;
234
235 return 0;
236}
237
239{
240 // absorb already open istream
241 if (type != tst_none)
242 close();
243 default_values();
244 is = &newis;
245 Origin = Token_Origin_Stream;
246 type = tst_istream;
247
248 return 0;
249}
250
252{
253 // Make a tokenstream from an internal existing string/buffer
254 const char *buf;
255 if (type != tst_none)
256 close();
257 default_values();
258 buf = (const char *)newbuffer;
259 buffer_length = newbuffer.length();
260 buffer = new char[buffer_length+1];
261 memmove(buffer,buf,buffer_length+1);
262 pos = 0;
263 Origin = Token_Origin_String;
264 type = tst_string;
265
266 return 0;
267}
268
269int EST_TokenStream::seek_end()
270{
271 // This isn't actually useful but people expect it
272 peeked_charp = FALSE;
273 peeked_tokp = FALSE;
274
275 switch (type)
276 {
277 case tst_none:
278 cerr << "EST_TokenStream unset" << endl;
279 return -1;
280 break;
281 case tst_file:
282 fseek(fp,0,SEEK_END);
283 p_filepos = ftell(fp);
284 return p_filepos;
285 case tst_pipe:
286 cerr << "EST_TokenStream seek on pipe not supported" << endl;
287 return -1;
288 break;
289 case tst_istream:
290 is->seekg(0,is->end);
291 p_filepos = is->tellg();
292 return p_filepos;
293 break;
294 case tst_string:
295 pos = buffer_length;
296 return pos;
297 default:
298 cerr << "EST_TokenStream: unknown type" << endl;
299 return -1;
300 }
301
302 return -1; // can't get here
303}
304
306{
307 peeked_charp = FALSE;
308 peeked_tokp = FALSE;
309
310 switch (type)
311 {
312 case tst_none:
313 cerr << "EST_TokenStream unset" << endl;
314 return -1;
315 break;
316 case tst_file:
317 p_filepos = position;
318 return fseek(fp,position,SEEK_SET);
319 case tst_pipe:
320 cerr << "EST_TokenStream seek on pipe not supported" << endl;
321 return -1;
322 break;
323 case tst_istream:
324 p_filepos = position;
325 is->seekg(position, is->beg);
326 return 0;
327 break;
328 case tst_string:
329 if (position >= pos)
330 {
331 pos = position;
332 return -1;
333 }
334 else
335 {
336 pos = position;
337 return 0;
338 }
339 break;
340 default:
341 cerr << "EST_TokenStream: unknown type" << endl;
342 return -1;
343 }
344
345 return -1; // can't get here
346
347}
348
349static int stdio_fread(void *buff,int size,int nitems,FILE *fp)
350{
351 // So it can find the stdio one rather than the TokenStream one
352 return fread(buff,size,nitems,fp);
353}
354
355int EST_TokenStream::fread(void *buff, int size, int nitems)
356{
357 // switching into binary mode for current position
358 int items_read;
359
360 // so we can continue to read afterwards
361 if (peeked_tokp)
362 {
363 cerr << "ERROR " << pos_description()
364 << " peeked into binary data" << endl;
365 return 0;
366 }
367
368 peeked_charp = FALSE;
369 peeked_tokp = FALSE;
370
371 switch (type)
372 {
373 case tst_none:
374 cerr << "EST_TokenStream unset" << endl;
375 return 0;
376 break;
377 case tst_file:
378 items_read = stdio_fread(buff,(size_t)size,(size_t)nitems,fp);
379 p_filepos += items_read*size;
380 return items_read;
381 case tst_pipe:
382 cerr << "EST_TokenStream fread pipe not yet supported" << endl;
383 return 0;
384 break;
385 case tst_istream:
386 is->read((char*)buff, (size_t) size*nitems);
387 return is->gcount()/size;
388 break;
389 case tst_string:
390 if ((buffer_length-pos)/size < nitems)
391 items_read = (buffer_length-pos)/size;
392 else
394 memcpy(buff,&buffer[pos],items_read*size);
395 pos += items_read*size;
396 return items_read;
397 default:
398 cerr << "EST_TokenStream: unknown type" << endl;
399 return EOF;
400 }
401
402 return 0; // can't get here
403
404}
405
407{
408 // close any files (if they were used)
409
410 switch (type)
411 {
412 case tst_none:
413 break;
414 case tst_file:
415 if (close_at_end)
416 fclose(fp);
417 case tst_pipe:
418 // close(fd);
419 break;
420 case tst_istream:
421 break;
422 case tst_string:
423 delete [] buffer;
424 buffer = 0;
425 break;
426 default:
427 cerr << "EST_TokenStream: unknown type" << endl;
428 break;
429 }
430
431 type = tst_none;
432 peeked_charp = FALSE;
433 peeked_tokp = FALSE;
434
435}
436
438{
439 // For paul, the only person I know who uses this
440
441 switch (type)
442 {
443 case tst_none:
444 break;
445 case tst_file:
446 fp = freopen(Origin,"rb",fp);
447 p_filepos = 0;
448 break;
449 case tst_pipe:
450 cerr << "EST_TokenStream: can't rewind pipe" << endl;
451 return -1;
452 break;
453 case tst_istream:
454 cerr << "EST_TokenStream: can't rewind istream" << endl;
455 break;
456 case tst_string:
457 pos = 0;
458 break;
459 default:
460 cerr << "EST_TokenStream: unknown type" << endl;
461 break;
462 }
463
464 linepos = 1;
465 peeked_charp = FALSE;
466 peeked_tokp = FALSE;
467 eof_flag = FALSE;
468
469 return 0;
470}
471
472EST_TokenStream & EST_TokenStream::operator >>(EST_Token &p)
473{
474 return get(p);
475}
476
477EST_TokenStream & EST_TokenStream::operator >>(EST_String &p)
478{
479 EST_Token t;
480
481 get(t);
482 p = t.string();
483 return *this;
484}
485
487{
488 tok = get();
489 return *this;
490}
491
493{
494 // Returns a concatenated token form here to next symbol that matches s
495 // including s (though not adding s on the result)
496 // Not really for the purist but lots of times very handy
497 // Note this is not very efficient
498 EST_String result;
499 EST_Token t;
500
501 for (result=EST_String::Empty; (t=get()) != s; )
502 {
503 result += t.whitespace() + t.prepunctuation() +
504 t.string() + t.punctuation();
505 if (eof())
506 {
507 cerr << "EST_TokenStream: end of file when looking for \"" <<
508 s << "\"" << endl;
509 break;
510 }
511 }
512
513 return EST_Token(result);
514}
515
517{
518 // Swallow the lot up to end of line
519 // assumes \n is a whitespace character
520
522
523 while (!eoln())
524 {
525 EST_Token &t=get();
526 result += t.whitespace() + t.prepunctuation();
527
528 if (quotes)
529 result += quote_string(t.string());
530 else
531 result += t.string();
532
533 result += t.punctuation();
534
535 if (eof())
536 {
537// cerr << "EST_TokenStream: end of file when looking for end of line"
538// << endl;
539 break;
540 }
541 }
542 // So that the next call works I have to step over the eoln condition
543 // That involves removing the whitespace upto and including the next
544 // \n in the peek token.
545
546 char *w = wstrdup(peek().whitespace());
547 int i;
548 for (i=0; w[i] != 0; i++)
549 if (w[i] == '\n') // maybe not portable
550 peek().set_whitespace(&w[i+1]);
551
552 wfree(w);
553
554 static EST_Token result_t;
555
556 result_t.set_token(result);
557
558 return result_t;
559}
560
561EST_Token &EST_TokenStream::must_get(EST_String expected, bool *ok)
562{
563 EST_Token &tok = get();
564
565 if (tok != expected)
566 {
567 if (ok != NULL)
568 {
569 *ok=FALSE;
570 return tok;
571 }
572 else
573 EST_error("Expected '%s' got '%s' at %s",
574 (const char *)expected,
575 (const char *)(EST_String)tok,
576 (const char *)pos_description());
577 }
578
579 if (ok != NULL)
580 *ok=TRUE;
581 return tok;
582}
583
584void EST_TokenStream::build_table()
585{
586 int i;
587 const char *p;
588 unsigned char c;
589
590 for (i=0; i<256; ++i)
591 p_table[i]=0;
592
593 for (p=WhiteSpaceChars; *p; ++p)
594 if (p_table[c=(unsigned char)*p])
595 EST_warning("Character '%c' has two classes, '%c' and '%c'",
596 *p, c, ' ');
597 else
598 p_table[c] = ' ';
599
600 for (p=SingleCharSymbols; *p; ++p)
601 if (p_table[c=(unsigned char)*p])
602 EST_warning("Character '%c' has two classes, '%c' and '%c'",
603 *p, p_table[c], '!');
604 else
605 p_table[c] = '@';
606
607 for (p=PunctuationSymbols; *p; ++p)
608 if (p_table[c=(unsigned char)*p] == '@')
609 continue;
610 else if (p_table[c])
611 EST_warning("Character '%c' has two classes, '%c' and '%c'",
612 *p, p_table[c], '.');
613 else
614 p_table[c] = '.';
615
616 for(p=PrePunctuationSymbols; *p; ++p)
617 if (p_table[c=(unsigned char)*p] == '@')
618 continue;
619 else if (p_table[c] == '.')
620 p_table[c] = '"';
621 else if (p_table[c])
622 EST_warning("Character '%c' has two classes, '%c' and '%c'",
623 *p, p_table[c], '$');
624 else
625 p_table[c] = '$';
626
627 p_table_wrong=0;
628}
629
630inline int EST_TokenStream::getpeeked_internal(void)
631{
632 peeked_charp = FALSE;
633 return peeked_char;
634}
635
636inline
637int EST_TokenStream::getch_internal()
638{
639 // Return next character in stream
640 if (EST_TokenStream::peeked_charp)
641 {
642 return getpeeked_internal();
643 }
644
645 switch (type)
646 {
647 case tst_none:
648 cerr << "EST_TokenStream unset" << endl;
649 return EOF;
650 break;
651 case tst_file:
652 p_filepos++;
653 {
654 char lc;
655 if (stdio_fread(&lc,1,1,fp) == 0)
656 return EOF;
657 else
658 return (int)lc;
659 }
660/* return getc(fp); */
661 case tst_pipe:
662 cerr << "EST_TokenStream pipe not yet supported" << endl;
663 return EOF;
664 break;
665 case tst_istream:
666 p_filepos++;
667 return is->get();
668 case tst_string:
669 if (pos < buffer_length)
670 {
671 p_filepos++;
672 return buffer[pos++];
673 }
674 else
675 return EOF;
676 default:
677 cerr << "EST_TokenStream: unknown type" << endl;
678 return EOF;
679 }
680
681 return EOF; // can't get here
682}
683
684int EST_TokenStream::getch(void)
685{
686 return getch_internal();
687}
688
689inline int EST_TokenStream::peekch_internal()
690{
691 // Return next character in stream (without reading it)
692
693 if (!peeked_charp)
694 peeked_char = getch_internal();
695 peeked_charp = TRUE;
696 return peeked_char;
697}
698
699
700int EST_TokenStream::peekch(void)
701{
702 return peekch_internal();
703
704}
705
706#define CLASS(C,CL) (p_table[(unsigned char)(C)]==(CL))
707
708#define CLASS2(C,CL1,CL2) (p_table[(unsigned char)(C)]==(CL1)||p_table[(unsigned char)(C)]==(CL2))
709
711{
712 if (peeked_tokp)
713 {
714 peeked_tokp = FALSE;
715 return current_tok;
716 }
717
718 if (p_table_wrong)
719 build_table();
720
721 char *word;
722 int c,i,j;
723
724 for (i=0; (CLASS(c=getch_internal(),' ') &&
725 ( c != EOF )); i++)
726 {
727 if (c == '\n') linepos++;
728 tok_wspace = check_extend_str(tok_wspace,i,&tok_wspacelen);
729 tok_wspace[i] = c;
730 }
731 tok_wspace[i] = '\0';
732
733 current_tok.init();
734
735 if (c != EOF)
736 {
737 current_tok.set_filepos(p_filepos-1);
738
739 if ((quotes) && // quoted strings (with escapes) are allowed
740 (c == quote))
741 {
742 for (i=0;
743 ((c = getch_internal()) != EOF)
744 ;)
745 {
746 if (c == quote)
747 break;
748 tok_stuff = check_extend_str(tok_stuff,i,&tok_stufflen);
749 if (c == escape)
750 c = getch_internal();
751 tok_stuff[i++] = c;
752 }
753 current_tok.set_quoted(TRUE);
754 }
755 else // standard whitespace separated tokens
756 {
757 for (i=0,tok_stuff[i++]=c;
758 (
759 !CLASS(c,'@') &&
760 !CLASS(c=peekch_internal(),' ') &&
761 !CLASS(c,'@') &&
762 ( c != EOF )) ;)
763 {
764 tok_stuff = check_extend_str(tok_stuff,i,&tok_stufflen);
765 // note, we must have peeked to get here.
766 tok_stuff[i++] = getpeeked_internal();
767 }
768 }
769 tok_stuff[i] = '\0';
770 // Are there any punctuation symbols at the start?
771 for (j=0;
772 ((j < i) && CLASS2(tok_stuff[j], '$', '"'));
773 j++);
774 if ((j > 0) && (j < i)) // there are
775 {
776 tok_prepuncs = check_extend_str(tok_prepuncs,j+1,&tok_prepuncslen);
777 memmove(tok_prepuncs,tok_stuff,j);
778 tok_prepuncs[j] = '\0';
779 current_tok.set_prepunctuation(tok_prepuncs);
780 word=&tok_stuff[j];
781 i-=j; // reduce size by number of prepuncs
782 }
783 else
784 {
786 word = tok_stuff;
787 }
788 // Are there any punctuation symbols at the end
789 for (j=i-1;
790 ((j > 0) && CLASS2(word[j],'.','"'));
791 j--);
792 if (word[j+1] != '\0')
793 {
794 current_tok.set_punctuation(&word[j+1]);
795 word[j+1] = '\0';
796 }
797 else
799
800 current_tok.set_token(word);
801 if (tok_wspace[0] == '\0') // feature paths will have null whitespace
803 else
804 current_tok.set_whitespace(tok_wspace);
805 }
806 else
807 {
808 current_tok.set_token(EST_String::Empty);
809 current_tok.set_whitespace(tok_wspace);
812 eof_flag = TRUE;
813 }
814
815 return current_tok;
816}
817
819{
820 // This doesn't really work if there are blank lines (and you want
821 // to know about them)
822
823 if ((peek().whitespace().contains("\n")) || eof())
824 return TRUE;
825 else
826 return FALSE;
827
828}
829
831{
832 if (!peeked_tokp) get();
833 peeked_tokp = TRUE;
834 return current_tok;
835}
836
837EST_String quote_string(const EST_String &s,
838 const EST_String &quote,
839 const EST_String &escape,
840 int force)
841{
842 // Quotes s always if force true, or iff s contains whitespace,
843 // quotes or escapes force is false
844 // Note quote and escape are assumed to be string of length 1
846 if ((force) ||
847 (s.contains(quote)) ||
848 (s.contains(escape)) ||
849 (s.contains(RXanywhitespace)) ||
850 (s.length() == 0))
851 {
852 // bigger than the quoted form could ever be
853 int i,j;
854 char *quoted = new char[s.length()*(quote.length()+escape.length())+
855 1+quote.length()+quote.length()];
856 quoted[0] = quote(0);
857 for (i=1,j=0; j < s.length(); j++,i++)
858 {
859 if (s(j) == quote(0))
860 quoted[i++] = escape(0);
861 else if (s(j) == escape(0))
862 quoted[i++] = escape(0);
863 quoted[i] = s(j);
864 }
865 quoted[i++] = quote(0);
866 quoted[i] = '\0';
867 quoted_form = quoted;
868 delete [] quoted;
869 return quoted_form;
870 }
871 else
872 return s;
873}
874
876{
877 return Origin+":"+itoString(linepos);
878}
static const EST_String Empty
Constant empty string.
Definition EST_String.h:111
int length(void) const
Length of string ({not} length of underlying chunk)
Definition EST_String.h:241
int contains(const char *s, int pos=-1) const
Does it contain this substring?
Definition EST_String.h:375
int restart(void)
Reset to start of file/string.
Definition EST_Token.cc:437
int eof()
end of file
Definition EST_Token.h:356
EST_Token & get()
get next token in stream
Definition EST_Token.cc:710
EST_Token get_upto(const EST_String &s)
get up to {\tt s} in stream as a single token.
Definition EST_Token.cc:492
int fread(void *buff, int size, int nitems) EST_WARN_UNUSED_RESULT
Reading binary data, (don't use peek() immediately beforehand)
Definition EST_Token.cc:355
const EST_String filename() const
The originating filename (if there is one)
Definition EST_Token.h:372
~EST_TokenStream()
will close file if appropriate for type
Definition EST_Token.cc:167
const EST_String pos_description()
A string describing current position, suitable for error messages.
Definition EST_Token.cc:875
int open_string(const EST_String &newbuffer)
open a \Ref{EST_TokenStream} for string rather than a file
Definition EST_Token.cc:251
EST_Token get_upto_eoln(void)
get up to {\tt s} in end of line as a single token.
Definition EST_Token.cc:516
int eoln()
end of line
Definition EST_Token.cc:818
EST_Token & peek(void)
peek at next token
Definition EST_Token.cc:830
void close(void)
Close stream.
Definition EST_Token.cc:406
int open(const EST_String &filename)
open a \Ref{EST_TokenStream} for a file.
Definition EST_Token.cc:200
int seek(int position)
seek, reposition file pointer
Definition EST_Token.cc:305
void set_prepunctuation(const EST_String &p)
set prepunction
Definition EST_Token.h:107
void set_whitespace(const EST_String &p)
set whitespace of token.
Definition EST_Token.h:99
void set_punctuation(const EST_String &p)
set (post) punctuation of token.
Definition EST_Token.h:103
const EST_String pos_description() const
A string describing current position, suitable for error messages.
Definition EST_Token.cc:107
void set_filepos(int c)
Set file position in original \Ref{EST_TokenStream}.
Definition EST_Token.h:176
void set_quoted(int q)
Note that this token was quoted (or not)
Definition EST_Token.h:167
void set_token(const EST_String &p)
set token from a string
Definition EST_Token.h:95