Mercurial > ~dholland > hg > ag > index.cgi

comparison anagram/agcore/ftpar.cpp @ 0:13d2b8934445

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Import AnaGram (near-)release tree into Mercurial.
author David A. Holland
date Sat, 22 Dec 2007 17:52:45 -0500
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:13d2b8934445
1 /*
2 * AnaGram, A System for Syntax Directed Programming
3 * Copyright 1993-2002 Parsifal Software. All Rights Reserved.
4 * See the file COPYING for license and usage terms.
5 *
6 * ftpar.cpp
7 */
8
9 #include "agarray.h"
10 #include "arrays.h"
11 #include "bpe3.h"
12 #include "cd.h"
13 #include "config.h"
14 #include "csexp.h"
15 #include "dict.h"
16 #include "ftpar.h"
17 #include "keyword.h"
18 #include "q1glbl.h"
19 #include "rule.h"
20 #include "token.h"
21 #include "tsd.h"
22
23 //#define INCLUDE_LOGGING
24 #include "log.h"
25
26
27 AgArray<unsigned> traceCounts;
28
29 //static dc_ref trace_count_display;
30
31
32 int precedes(cint a, cint b) {
33 if (a.y < b.y) return 1;
34 if (a.y > b.y) return 0;
35 if (a.x < b.x) return 1;
36 return 0;
37 }
38
39
40 FtParser::FtParser(AgString t)
41 : text(t)
42 , state(t.pointer())
43 , initialStack(0)
44 , lookAhead(state.pointer)
45 , endPointer(state.pointer +
46 (state.pointer != 0 ? strlen((const char *) state.pointer) : 0))
47 , inputToken(0)
48 , nNullShifts(0)
49 , processState(ready)
50 , ruleToReduce(0)
51 , reductionSelection(0)
52 , stackChanged(*this)
53 , testingKeyword(0)
54 {
55 LOGSECTION("FtParser::FtParser");
56 LOGV((int) state.pointer);
57 LOGV((int) lookAhead);
58 getToken();
59 if (!traceCounts.exists()) {
60 traceCounts = AgArray<unsigned>(nforms_base + 1);
61 memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
62 /*
63 trace_count_display =
64 dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
65 */
66 }
67 }
68
69 FtParser::FtParser()
70 : initialStack(0)
71 , lookAhead(0)
72 , inputToken(0)
73 , nNullShifts(0)
74 , processState(ready)
75 , ruleToReduce(0)
76 , reductionSelection(0)
77 , stackChanged(*this)
78 , testingKeyword(0)
79 {
80 LOGSECTION("FtParser::FtParser");
81 LOGV((int) state.pointer);
82 LOGV((int) lookAhead);
83 if (!traceCounts.exists()) {
84 traceCounts = AgArray<unsigned>(nforms_base + 1);
85 memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
86 /*
87 trace_count_display =
88 dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
89 */
90 }
91 }
92
93 FtParser::FtParser(tsd *initialStack_)
94 : initialStack(initialStack_ ? copy_tuple_set(initialStack_) : 0)
95 , lookAhead(0)
96 , inputToken(0)
97 , nNullShifts(0)
98 , processState(ready)
99 , ruleToReduce(0)
100 , reductionSelection(0)
101 , stackChanged(*this)
102 , testingKeyword(0)
103 {
104 LOGSECTION("FtParser::FtParser");
105 LOGV((int) state.pointer);
106 LOGV((int) lookAhead);
107 LOGV((int) initialStack);
108 if (initialStack) {
109 LOGV(initialStack->nt);
110 for (unsigned i = 0; i < initialStack->nt; i++) {
111 unsigned sn, tn;
112 xtx(initialStack,i, &sn, &tn);
113 stateStack.push(State(sn,tn));
114 }
115 state = stateStack.pop();
116 }
117 LOGV(stateStack.size());
118 if (!traceCounts.exists()) {
119 traceCounts = AgArray<unsigned>(nforms_base + 1);
120 memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
121 /*
122 trace_count_display =
123 dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
124 */
125 }
126 }
127
128 FtParser::~FtParser() {
129 if (initialStack) {
130 delete_tsd(initialStack);
131 }
132 }
133
134 FtParser &FtParser::reset() {
135 LOGSECTION("FtParser::reset");
136 stateStack.discardData();
137 auxStack.discardData();
138 transStack.discardData();
139 LOGV(auxStack.size());
140 LOGV((int) initialStack);
141 inputToken = 0;
142 if (initialStack) {
143 LOGV(initialStack->nt);
144 for (unsigned i = 0; i < initialStack->nt; i++) {
145 unsigned sn, tn;
146 xtx(initialStack,i, &sn, &tn);
147 stateStack.push(State(sn, tn));
148 }
149 state = stateStack.pop();
150 }
151 else {
152 state = State(text.pointer());
153 lookAhead = state.pointer;
154 endPointer = state.pointer +
155 (state.pointer != 0 ? strlen((const char *) state.pointer) : 0);
156 getToken();
157 }
158 LOGV(stateStack.size());
159 reductionState = State();
160 nNullShifts = 0;
161 processState = ready;
162 //if ((dc *)displayControl) {
163 // displayControl->des->d_size.y = stateStack.size();
164 //}
165 stackChanged(stateStack.size() + 1);
166 return *this;
167 }
168
169 void FtParser::track(void) {
170 LOGSECTION("FtParser::track");
171 LOGV((int) state.pointer);
172 LOGV((int) lookAhead);
173 if (lookAhead != 0) {
174 while (state.pointer < lookAhead) {
175 switch (*state.pointer++) {
176 case '\n':
177 if (*state.pointer) state.column = state.charPos = 0, state.line++;
178 case '\f':
179 break;
180 case '\t':
181 state.column += tab_spacing - state.column % tab_spacing;
182 state.charPos++;
183 break;
184 default:
185 state.column++;
186 state.charPos++;
187 }
188 }
189 }
190 state.token = inputToken = 0;
191 auxStack.discardData();
192 transStack.discardData();
193 LOGV(auxStack.size());
194 nNullShifts = 0;
195 LOGV(state.line) LCV(state.column);
196 }
197
198 void FtParser::shiftTerminalAndAccept(void) {
199 LOGSECTION("FtParser::shiftTerminalAndAccept");
200 processState = finished;
201 track();
202 }
203
204 void FtParser::shiftTerminal() {
205 LOGSECTION("FtParser::shiftTerminal");
206 LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
207 LOGV(stateStack.size());
208 state.token = map_state_number[actionParameter].char_token;
209 stateStack.push(state);
210 state.number = actionParameter;
211 track();
212 LOGV(state.token);
213 LOGV(state.number) LCV((int) state.pointer);
214 LOGV(stateStack.size());
215 }
216
217 void FtParser::requestSelection(int actionParameter) {
218 LOGSECTION("FtParser::requestSelection");
219 processState = selectionRequired;
220 ruleToReduce = actionParameter;
221 reductionSelection = 0;
222 while (!validSelection(reductionSelection, reductionState.number)) {
223 reductionSelection++;
224 }
225 LOGV(ruleToReduce);
226 LOGV(reductionIndex);
227 LOGV(state.number) LCV((int) state.pointer);
228 LOGV(reductionState.number);
229 reductionState.token = ibnfs[ibnfb[ruleToReduce]];
230 LOGV(stateStack.size());
231 }
232
233 /*
234 * State stack discipline
235 * reduce
236 * If n is the length of the rule, n states are popped from the
237 * state stack.
238 * If n > 0, the state number becomes the state number of the
239 * last state popped.
240 *
241 * shiftTerminalAndReduce
242 * If n is the length of the rule, n-1 states are popped from
243 * the state stack.
244 * If n > 1, the state number becomes the state number of the
245 * last state popped.
246 *
247 * shiftNonterminalAndReduce
248 * If n is the length of the rule, n-1 states are popped from
249 * the state stack.
250 * If n > 1, the state number becomes the state number of the
251 * last state popped.
252 *
253 * shiftNull
254 * The current state is pushed to the state stack and the
255 * new state number is given by the action parameter.
256 *
257 * shiftNonterminal
258 * The current state is pushed to the state stack and the
259 * new state number is given by the action parameter
260 */
261
262 void FtParser::shiftTerminalAndReduce() {
263 LOGSECTION("FtParser::shiftTerminalAndReduce");
264 LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
265 LOGV(location());
266 LOGV(actionParameter);
267 //form_number_map *fp = &map_form_number[actionParameter];
268 Rule rule(actionParameter);
269 RuleDescriptor &ruleDescriptor(rule);
270 //ruleLength = rule->length();
271 ruleLength = ruleDescriptor.length();
272 LOGV(ruleLength);
273 if (actionParameter <= nforms_base) {
274 traceCounts[actionParameter]++;
275 }
276 int nStackedStates = ruleLength - 1;
277 if (nStackedStates > 0) {
278 reductionState = stateStack[stateStack.size() - nStackedStates];
279 }
280 else {
281 reductionState = state;
282 }
283 track();
284 if (ibnfn[actionParameter] > 1) {
285 reductionIndex = stateStack.size() - (ruleLength - 1);
286 assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
287 requestSelection(actionParameter);
288 return;
289 }
290 assert((unsigned)nStackedStates <= stateStack.size());
291 stateStack.discardData(nStackedStates);
292 //reductionState.token = rule->prim_tkn;
293 reductionState.token = ruleDescriptor.prim_tkn;
294 state.number = reductionState.number;
295 dispatchReductionToken();
296 LOGV(location());
297 LOGV(state.number);
298 LOGV(state.token) LCV((int) state.pointer);
299 auxStack.discardData();
300 transStack.discardData();
301 LOGV(auxStack.size());
302 }
303
304 void FtParser::reduce(void) {
305 LOGSECTION("FtParser::reduce");
306 LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
307 //form_number_map *fp = &map_form_number[actionParameter];
308 Rule rule(actionParameter);
309 RuleDescriptor &ruleDescriptor(rule);
310 //ruleLength = rule->length();
311 ruleLength = ruleDescriptor.length();
312 LOGV(actionParameter);
313 LOGV(ruleLength);
314 LOGV(stateStack.size());
315 if (actionParameter <= nforms_base) {
316 traceCounts[actionParameter]++;
317 }
318 if (ruleLength) {
319 reductionState = stateStack[stateStack.size() - ruleLength];
320 }
321 else {
322 reductionState = state;
323 }
324 if (ibnfn[actionParameter] > 1) {
325 reductionIndex = stateStack.size() - ruleLength;
326 assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
327 requestSelection(actionParameter);
328 return;
329 }
330 int k = ruleLength;
331 transStack.push(Transaction(k, state));
332 LOGV(transStack.size());
333 LOGV(k) LCV(state.number) LCV(state.token);
334 LOGV(nNullShifts);
335 while (k--) {
336 LOGV(stateStack.top().number) LCV(stateStack.top().token)
337 LCV((int) state.pointer);
338 auxStack.push(stateStack.pop());
339 }
340 LOGV(auxStack.size());
341 //reductionState.token = rule->prim_tkn;
342 reductionState.token = ruleDescriptor.prim_tkn;
343 state.number = reductionState.number;
344 LOGV(reductionState.token);
345 LOGV(stateStack.size());
346 dispatchReductionToken();
347 LOGV(stateStack.size());
348 if (lookAhead) {
349 lookAhead = state.pointer;
350 }
351 state.token = 0;
352 }
353
354 void FtParser::skip(void) {
355 LOGSECTION("FtParser::skip");
356 LOGV(state.number) LCV(state.token);
357 if (actionParameter <= nforms_base) {
358 traceCounts[actionParameter]++;
359 }
360 track();
361 }
362
363 void FtParser::shiftNull(void) {
364 LOGSECTION("FtParser::shiftNull");
365 LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
366 transStack.push(Transaction(-1, state));
367 LOGV(transStack.size());
368 state.token = map_state_number[actionParameter].char_token;
369 stateStack.push(state);
370 state.number = actionParameter;
371 if (lookAhead) {
372 lookAhead = state.pointer;
373 }
374 state.token = 0;
375 }
376
377 void FtParser::error() {
378 LOGSECTION("FtParser::error");
379 LOGV(stateStack.size());
380 LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
381 LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
382 LOGV(auxStack.size());
383 int k = transStack.size();
384 LOGV(transStack.size());
385 while (k--) {
386 Transaction trans = transStack.pop();
387 int n = trans.count;
388 LOGV(trans.count) LCV(trans.state.number) LCV(trans.state.token);
389 while (n < 0) {
390 stateStack.pop();
391 n++;
392 }
393 while (n > 0) {
394 stateStack.push(auxStack.pop());
395 n--;
396 }
397 state.number = trans.state.number;
398 state.token = trans.state.token;
399 LOGV(trans.count) LCV(state.number) LCV(state.token);
400 }
401 auxStack.discardData();
402 transStack.discardData();
403 processState = syntaxError;
404 if (lookAhead) {
405 lookAhead = state.pointer;
406 }
407 state.token = 0;
408 }
409
410
411 void FtParser::accept(void) {
412 LOGSECTION("FtParser::accept");
413 state.number = stateStack.top().number;
414 state.token = stateStack.top().token;
415 stateStack.pop();
416 LOGV(stateStack.size());
417 processState = finished;
418 }
419
420 int FtParser::shiftNonterminal() {
421 LOGSECTION("FtParser::shiftNonterminal");
422 LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
423 transStack.push(Transaction(-1, state));
424 LOGV(transStack.size());
425 LOGV(actionParameter) LCV(nstates);
426 reductionState.token = map_state_number[actionParameter].char_token;
427 #ifdef INCLUDE_LOGGING
428 ics();
429 atkn(reductionState.token);
430 LOGV(reductionState.token) LCV(string_base);
431 rcs();
432 #endif
433 stateStack.push(reductionState);
434 state.number = actionParameter;
435 LOGV(stateStack.size());
436 LOGV(state.number) LCV((int) state.pointer);
437 return 0;
438 }
439
440 int FtParser::shiftNonterminalAndReduce() {
441 LOGSECTION("FtParser::shiftNonterminalAndReduce");
442 LOGV(location());
443 LOGV(reductionState.number) LCV(reductionState.token);
444 //form_number_map *fp = &map_form_number[actionParameter];
445 Rule rule(actionParameter);
446 RuleDescriptor &ruleDescriptor(rule);
447 //ruleLength = rule->length();
448 ruleLength = ruleDescriptor.length();
449 LOGS("rule number") LCV(actionParameter);
450 LOGV(ruleLength);
451 LOGV(rule->prim_tkn);
452 LOGV(stateStack.size());
453 if (actionParameter <= nforms_base) {
454 traceCounts[actionParameter]++;
455 }
456 int nStackedStates = ruleLength - 1;
457 if (nStackedStates > 0) {
458 reductionState = stateStack[stateStack.size() - nStackedStates];
459 }
460 if (ibnfn[actionParameter] > 1) {
461 reductionIndex = stateStack.size() - (ruleLength - 1);
462 assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
463 requestSelection(actionParameter);
464 return 0;
465 }
466 transStack.push(Transaction(nStackedStates, state));
467 LOGV(transStack.size());
468 LOGV(nStackedStates) LCV(state.number) LCV(state.token);
469 LOGV(auxStack.size());
470 while (nStackedStates--) {
471 LOGV(stateStack.top().number) LCV(stateStack.top().token)
472 LCV((int) state.pointer);
473 auxStack.push(stateStack.pop());
474 }
475 LOGV(auxStack.size());
476 //reductionState.token = rule->prim_tkn;
477 reductionState.token = ruleDescriptor.prim_tkn;
478 state.number = reductionState.number;
479 LOGV(location());
480 LOGV(stateStack.size());
481 return 1;
482 }
483
484 int FtParser::shiftNonterminalAndAccept() {
485 LOGSECTION("FtParser::shiftNonTerminalAndAccept");
486 state.number = reductionState.number;
487 state.token = reductionState.token;
488 LOGV(state.number);
489 LOGV(state.token) LCV((int) state.pointer);
490 processState = finished;
491 return 0;
492 }
493
494 int (FtParser::*FtParser::nonterminalAction[4])() = {
495 &FtParser::shiftNonterminalAndAccept,
496 &FtParser::shiftNonterminal,
497 &FtParser::shiftNonterminalAndReduce,
498 &FtParser::shiftNonterminalAndReduce
499 };
500
501 void (FtParser::*FtParser::terminalAction[11])() = {
502 &FtParser::shiftTerminalAndAccept,
503 &FtParser::shiftTerminal,
504 &FtParser::shiftTerminalAndReduce,
505 &FtParser::shiftTerminalAndReduce,
506 &FtParser::reduce,
507 &FtParser::reduce,
508 &FtParser::accept,
509 &FtParser::error,
510 &FtParser::shiftNull,
511 &FtParser::skip,
512 &FtParser::skip
513 };
514
515 static int different(const unsigned char *s1, const unsigned char *s2, unsigned n) {
516 LOGSECTION("different");
517 LOGV(n);
518 //if (n > strlen((const char *)s2)) return 1;
519 if (case_sensitive) {
520 return strncmp((const char *)s1, (const char *) s2, n);
521 }
522 while (n--) {
523 if (agToUpper(*s1++) != agToUpper(*s2++)) {
524 return 1;
525 }
526 }
527 return 0;
528 }
529
530
531 Token FtParser::keyToken(void) {
532 LOGSECTION("FtParser::keyToken");
533 int matchLength = 0;
534 //int key = 0;
535 Keyword key;
536 int keyListNumber = map_state_number[state.number].key_list;
537 if (keyListNumber == 0) {
538 return Token();
539 }
540
541 int *keyList = dict_str(key_list_dict, keyListNumber);
542 int nKeys = *keyList++ - 1;
543 LOGV(lookAhead) LCV(nKeys);
544 while (nKeys--) {
545 Keyword keyNumber = map_token_number[*keyList++].key;
546 KeywordDescriptor &keyDescriptor(keyNumber);
547 //unsigned char *keyString = (unsigned char *)keyNumber->string.pointer();
548 unsigned char *keyString = (unsigned char *)keyDescriptor.string.pointer();
549 //AgString keyString = keyNumber->string;
550 int length = strlen((const char *) keyString);
551 //int length = keyString.size();
552 if (length <= matchLength) {
553 continue;
554 }
555 if ((lookAhead + length) > endPointer) {
556 continue;
557 }
558 if (different(keyString, lookAhead, length)) {
559 continue;
560 }
561 //if (different((unsigned char *)keyString.pointer(), lookAhead, length)) {
562 // continue;
563 //}
564 //int charSetNumber = keyNumber->reserve;
565 int charSetNumber = keyDescriptor.reserve;
566 if (charSetNumber) {
567 int *reservedCharSet = dict_str(char_set_dict, charSetNumber);
568 int nReservedCharSet = *reservedCharSet++ - 1;
569 if (lookAhead + length < endPointer) {
570 unsigned char fc = state.pointer[length];
571 while (nReservedCharSet && fc != *reservedCharSet++) {
572 nReservedCharSet--;
573 }
574 if (nReservedCharSet) {
575 continue;
576 }
577 }
578 }
579 matchLength = length;
580 key = keyNumber;
581 }
582 LOGV((int) lookAhead) LCV(matchLength);
583 LOGV(key) LCV(testingKeyword);
584 if (key.isNotNull() && (int) key != testingKeyword) {
585 lookAhead = state.pointer + matchLength;
586 LOGV(key->token_number);
587 return key->token_number;
588 }
589 return Token();
590 }
591
592
593 void FtParser::getToken() {
594 LOGSECTION("FtParser::getToken");
595 LOGV(stateStack.size());
596 LOGV(state.number) LCV((int) state.pointer);
597 LOGV((int) lookAhead);
598 LOGV((int) endPointer);
599 LOGV(inputToken);
600 if (inputToken) {
601 state.token = inputToken;
602 }
603 else if (lookAhead != 0 && lookAhead >= endPointer) {
604 if (text.exists() && eof_token) {
605 token_number_map &eofTokenMap = map_token_number[eof_token];
606 //if (map_token_number[eof_token].non_terminal_flag) {
607 if (eofTokenMap.non_terminal_flag) {
608 //unsigned charSet = map_token_number[eof_token].token_set_id;
609 unsigned charSet = eofTokenMap.token_set_id;
610 unsigned *list = (unsigned *) dict_str(char_set_dict,charSet);
611 int character = list[1];
612 LOGV(charSet);
613 LOGV(character);
614 state.token = map_char_number[character].token_number;
615 }
616 else {
617 state.token = eof_token;
618 }
619 LOGV(state.token) LCV((int) state.pointer);
620 }
621 else {
622 processState = unexpectedEndOfFile;
623 state.token = 0;
624 }
625 }
626 else if (lookAhead != 0) {
627 state.token = keyToken();
628 LOGV(state.token) LCV((int) state.pointer);
629 if (state.token == 0) {
630 state.token =
631 map_char_number[*lookAhead++ - min_char_number].token_number;
632 }
633 }
634 LOGV(state.token) LCV((int) state.pointer);
635 LOGV(auxStack.size());
636 }
637
638 /*
639 unsigned FtParser::inspectToken() {
640 LOGSECTION("FtParser::inspectToken");
641 LOGV((int) lookAhead);
642 LOGV((int) endPointer);
643 unsigned token = 0;
644 if (inputToken) {
645 token = inputToken;
646 }
647 else if (lookAhead !=0 && lookAhead >= endPointer) {
648 if (text.exists() && eof_token) {
649 if (map_token_number[eof_token].non_terminal_flag) {
650 unsigned charSet = map_token_number[eof_token].token_set_id;
651 unsigned *list = (unsigned *) dict_str(char_set_dict,charSet);
652 int character = list[1];
653 LOGV(charSet);
654 LOGV(character);
655 token = map_char_number[character].token_number;
656 }
657 else {
658 token = eof_token;
659 }
660 LOGV(token);
661 }
662 else {
663 token = 0;
664 }
665 }
666 else if (lookAhead != 0) {
667 const unsigned char *save = lookAhead;
668 token = keyToken();
669 lookAhead = save;
670 LOGV(token);
671 if (token == 0) {
672 token = map_char_number[*lookAhead - min_char_number].token_number;
673 }
674 }
675 LOGV(token);
676 LOGV(auxStack.size());
677 return token;
678 }
679 */
680
681 void FtParser::dispatchReductionToken() {
682 LOGSECTION("FtParser::dispatchReductionToken");
683 unsigned k;
684 state_number_map *sp;
685
686 do {
687 LOGV(reductionState.number);
688 LOGV(reductionState.token);
689 sp = &map_state_number[reductionState.number];
690 unsigned *tokenPointer = lstptr(*sp,t_actions);
691 for (k = sp->n_actions; k && tokenPointer[--k] != reductionState.token;) {
692 /* nothing */
693 }
694 LOGV(k) LCV(tokenPointer[k]);
695 assert(tokenPointer[k] == reductionState.token);
696 actionParameter = lstptr(*sp,p_actions)[k];
697 LOGV(k) LCV(actionParameter);
698 if (k == 0) {
699 shiftNonterminal();
700 processState = selectionError;
701 stackChanged(stateStack.size()+1);
702 return;
703 }
704 } while ((this->*(nonterminalAction[lstptr(*sp,a_actions)[k]]))());
705 stackChanged(stateStack.size()+1);
706 }
707
708 void FtParser::completeReduction(int token) {
709 LOGSECTION("FtParser::completeReduction(int)");
710 int k = stateStack.size() - reductionIndex;
711 assert((unsigned) k <= (unsigned) stateStack.size());
712 LOGV(k);
713 stateStack.discardData(k);
714 reductionState.token = token;
715 LOGV(token);
716 processState = running;
717 dispatchReductionToken();
718 if (processState == running &&
719 state.pointer == lookAhead &&
720 state.token == 0) {
721 getToken();
722 }
723 nNullShifts = 0;
724 auxStack.discardData();
725 transStack.discardData();
726 LOGV(auxStack.size());
727 }
728
729 void FtParser::completeReduction() {
730 LOGSECTION("FtParser::completeReduction()");
731 int token = ibnfs[ibnfb[ruleToReduce]+reductionSelection];
732 completeReduction(token);
733 }
734
735 void FtParser::parseAction() {
736 LOGSECTION("FtParser::parseAction");
737 state_number_map *sp = &map_state_number[state.number];
738 unsigned *tokenPointer = lstptr(*sp, t_actions);
739
740 if (processState < running) {
741 processState = running;
742 }
743 int k = 0;
744 while (tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
745 k++;
746 }
747
748 actionParameter = lstptr(*sp, p_actions)[k];
749 ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
750
751 if (processState <= running && state.token == 0) {
752 getToken();
753 }
754 stackChanged(stateStack.size()+1);
755 }
756
757 void FtParser::stepToken(unsigned token) {
758 LOGSECTION("FtParser::stepToken");
759 LOGV(state.number);
760 LOGV(state.token) LCV((int) state.pointer);
761 lookAhead = state.pointer = 0;
762 inputToken = token;
763 processState = running;
764
765 if (map_token_number[token].non_terminal_flag) {
766 reductionState = state;
767 reductionState.token = token;
768 state.token = inputToken = 0;
769 dispatchReductionToken();
770 transStack.discardData();
771 auxStack.discardData();
772 if (processState <= running && state.token == 0) {
773 getToken();
774 }
775 stackChanged(stateStack.size());
776 return;
777 }
778 inputToken = token;
779 state_number_map *sp = &map_state_number[state.number];
780 unsigned *tokenPointer = lstptr(*sp, t_actions);
781 state.token = token;
782
783 int k = 0;
784 while(tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
785 k++;
786 }
787
788 actionParameter = lstptr(*sp, p_actions)[k];
789 ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
790 LOGV(processState) LCV(state.token);
791 if (processState <= running && state.token == 0) {
792 getToken();
793 }
794 stackChanged(stateStack.size()+1);
795 }
796
797 void FtParser::parseToken(unsigned token) {
798 LOGSECTION("FtParser::parseToken");
799 LOGV(state.number) LCV(token);
800 LOGV(state.token) LCV((int) state.pointer);
801 lookAhead = state.pointer = 0;
802 inputToken = token;
803 processState = token ? running : syntaxError;
804 if (processState == syntaxError) {
805 return;
806 }
807
808 if (map_token_number[token].non_terminal_flag) {
809 reductionState = state;
810 reductionState.token = token;
811 state.token = inputToken = 0;
812 dispatchReductionToken();
813 if (processState <= running && state.token == 0) {
814 getToken();
815 }
816 return;
817 }
818 inputToken = state.token = token;
819
820 while (processState <= running && inputToken != 0) {
821 state_number_map *sp = &map_state_number[state.number];
822 unsigned *tokenPointer = lstptr(*sp, t_actions);
823 int k = 0;
824 while(tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
825 k++;
826 }
827 LOGV(k) LCV(sp->n_actions);
828 assert((unsigned) k < sp->n_actions);
829 actionParameter = lstptr(*sp, p_actions)[k];
830 ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
831 LOGV(processState) LCV(state.token);
832 if (processState <= running && inputToken != 0) {
833 state.token = inputToken;
834 }
835 }
836 if (processState <= running && state.token == 0) {
837 getToken();
838 }
839 stackChanged(stateStack.size()+1);
840 }
841
842
843
844 int FtParser::validToken(unsigned token, unsigned sn) {
845 LOGSECTION("FtParser::validToken");
846 state_number_map *sp = &map_state_number[sn];
847 unsigned *tokenPointer = lstptr(*sp, t_actions);
848
849 int k = 0;
850 while (tokenPointer[k] && tokenPointer[k] != token && tokenPointer[k]) {
851 k++;
852 }
853 if (k == 0 && lstptr(*sp, a_actions)[k] == pe_syn_error) {
854 return 0;
855 }
856 LOGV(sn);
857 LOGV(token);
858 LOGV(k);
859 LOGV(tokenPointer[k]);
860 return 1;
861 }
862
863 int FtParser::validSelection(unsigned selection, unsigned sn) {
864 LOGSECTION("FtParser::validSelection");
865 state_number_map *sp = &map_state_number[sn];
866 unsigned *tokenPointer = lstptr(*sp, t_actions);
867
868 unsigned token = ibnfs[ibnfb[ruleToReduce] + selection];
869
870 int k;
871 for (k = sp->n_actions; k && tokenPointer[--k] != token; ) {
872 LOGV(k);
873 LOGV(tokenPointer[k]);
874 }
875 LOGV(sn);
876 LOGV(token);
877 LOGV(k);
878 LOGV(tokenPointer[k]);
879 return k != 0;
880 }
881
882 FtParser &FtParser::parseTo(unsigned char *target) {
883 LOGSECTION("FtParser::parseTo");
884 int backup = 0;
885 while (stateStack.size() && state.pointer <= target) {
886 stateStack.pop(state);
887 backup = 1;
888 }
889 if (backup) {
890 processState = running;
891 lookAhead = state.pointer;
892 auxStack.discardData();
893 transStack.discardData();
894 LOGV(auxStack.size());
895 nNullShifts = 0;
896 getToken();
897 }
898 else if (processState == finished) {
899 reset();
900 }
901 if (processState < running) {
902 processState = running;
903 }
904 LOGV(state.token) LCV((int) state.pointer);
905 while (processState <= running && state.pointer < target) {
906 parseAction();
907 }
908 //if ((dc *)displayControl) {
909 // displayControl->des->d_size.y = stateStack.size()+1;
910 //}
911 stackChanged(stateStack.size()+1);
912 return *this;
913 }
914
915 FtParser &FtParser::parse() {
916 if (processState < running) {
917 processState = running;
918 }
919 while (processState <= running) {
920 parseAction();
921 }
922 //if ((dc *)displayControl) {
923 // displayControl->des->d_size.y = stateStack.size()+1;
924 //}
925 stackChanged(stateStack.size()+1);
926 return *this;
927 }
928
929 FtParser &FtParser::parse(const char *fragment) {
930 LOGSECTION("FtParser::parse(const char *)");
931 LOGV(fragment);
932 lookAhead = state.pointer = (const unsigned char *) fragment;
933 inputToken = 0;
934 endPointer = (const unsigned char *) fragment + strlen(fragment);
935 getToken();
936 LOGV(processState) LCV(inputToken);
937
938 while (processState <= running) {
939 parseAction();
940 LOGV(state.pointer);
941 }
942 //if ((dc *)displayControl) {
943 // displayControl->des->d_size.y = stateStack.size()+1;
944 //}
945 stackChanged(stateStack.size()+1);
946 if (processState == unexpectedEndOfFile) {
947 processState = ready;
948 }
949 return *this;
950 }
951
952 FtParser &FtParser::prime(const char *fragment) {
953 LOGSECTION("FtParser::parse(const char *)");
954 LOGV(fragment);
955 lookAhead = state.pointer = (const unsigned char *) fragment;
956 endPointer = (const unsigned char *) fragment + strlen(fragment);
957 getToken();
958 LOGV(processState);
959 stackChanged(stateStack.size()+1);
960 return *this;
961 }
962
963 FtParser &FtParser::parseTo(cint *loc) {
964 LOGSECTION("FtParser::parseTo");
965 LOGV(*loc);
966 LOGV(stateStack.size());
967 LOGV(location());
968 int backup = 0;
969 LOGV(processState);
970 while (stateStack.size()
971 && (loc->y < state.line
972 || (loc->y == state.line && loc->x <= state.charPos))) {
973 stateStack.pop(state);
974 backup = 1;
975 }
976 LOGV(*loc) LCV(location());
977 LOGV(precedes(*loc, location()));
978 if (backup) {
979 processState = running;
980 lookAhead = state.pointer;
981 getToken();
982 }
983 else if (precedes(*loc, location())) {
984 reset();
985 }
986 LOGV(backup);
987 LOGV(processState);
988 LOGV(stateStack.size());
989 LOGV(state.number) LCV((int) state.pointer);
990 LOGV(location());
991 if (processState < running) {
992 processState = running;
993 }
994 while (processState == running &&
995 (state.line < loc->y || (state.line == loc->y &&
996 state.charPos < loc->x))) {
997 parseAction();
998 }
999 LOGV(location());
1000 //if ((dc *)displayControl) {
1001 // displayControl->des->d_size.y = stateStack.size()+1;
1002 //}
1003 stackChanged(stateStack.size()+1);
1004 return *this;
1005 }
1006
1007 FtParser &FtParser::step() {
1008 LOGSECTION("FtParser::step");
1009 LOGV(location());
1010 if (processState < running) {
1011 processState = running;
1012 }
1013 if (processState == running) {
1014 parseAction();
1015 }
1016 //if ((dc *)displayControl) {
1017 // displayControl->des->d_size.y = stateStack.size()+1;
1018 //}
1019 stackChanged(stateStack.size()+1);
1020 return *this;
1021 }
1022
1023 FtParser &FtParser::step(char *fragment) {
1024 LOGSECTION("FtParser::step(const char *)");
1025 LOGV(fragment);
1026 lookAhead = state.pointer = (unsigned char *) fragment;
1027 endPointer = (unsigned char *) fragment + strlen(fragment);
1028 getToken();
1029 LOGV(processState);
1030 if (processState <= running) {
1031 parseAction();
1032 }
1033 //if ((dc *)displayControl) {
1034 // displayControl->des->d_size.y = stateStack.size()+1;
1035 //}
1036 stackChanged(stateStack.size()+1);
1037 if (processState <= running && state.token == 0) {
1038 getToken();
1039 }
1040 if (processState == unexpectedEndOfFile) {
1041 processState = ready;
1042 }
1043 return *this;
1044 }
1045
1046 const char *FtParser::processStateText[] = {
1047 "Ready",
1048 "Ready", //running,
1049 "Parse complete", //finished,
1050 "Syntax error", //syntaxError,
1051 "Unexpected end of file", //unexpectedEndOfFile,
1052 "Select reduction token", //selectionRequired
1053 "Selection error"
1054 };
1055
1056 #if 0 /* unused */
1057 tsd *x1x_new(pcb_type *pcb) {
1058 int sx, sn, tn, /*fn,*/ fx;
1059 int *items;
1060 int nitems;
1061 tsd *isl = init_tsd(4);
1062
1063 ok_ptr(pcb);
1064 sn = PCB.s.sn;
1065 tn = PCB.token_number;
1066 if (PCB.exit_flag) {
1067 tn = 0;
1068 }
1069 sx = PCB.ssx;
1070 if (PCB.reduction_token) {
1071 sx -= PCB.rule_length;
1072 }
1073
1074 {
1075 tuple_dict *d;
1076 d = xis(sn);
1077 items = d->text;
1078 nitems = d->nsx;
1079 items += 2*nitems;
1080 while (nitems--) {
1081 fx = *--items;
1082 Rule rule = *--items;
1083 if (tn == 0 ||
1084 fx >= rule->non_vanishing_length ||
1085 rule.token(fx).isExpansionToken(tn)) {
1086 //x2(Rule(fn)->token(fx), tn))
1087 //x2(lstptr(map_form_number[fn],tokens)[fx], tn))
1088 at(isl,sx,sn,(int) rule,fx);
1089 }
1090 }
1091 delete_tuple_dict(d);
1092 }
1093
1094 while (sx-- > 0) {
1095 tuple_dict *d;
1096 tn = map_state_number[sn].char_token;
1097 sn = PCB.ss[sx].sn;
1098 d = xis(sn);
1099 items = d->text;
1100 nitems = d->nsx;
1101 items += 2*nitems;
1102 while (nitems--) {
1103 fx = *--items;
1104 Rule rule = *--items;
1105 if (fx >= rule->length()) {
1106 continue;
1107 }
1108 if (x3(isl, sx, (int)rule, fx)) {
1109 at(isl, sx, sn, (int) rule, fx);
1110 }
1111 }
1112 delete_tuple_dict(d);
1113 }
1114 return isl;
1115 }
1116 #endif /* 0 - unused */
1117
1118 tsd *FtParser::x1x_new() {
1119 LOGSECTION("FtParser::x1x_new");
1120 int sx, sn, fn, fx;
1121 tsd *isl = init_tsd(4);
1122
1123 sn = state.number;
1124 Token tn = state.token;
1125 if (processState == selectionRequired) {
1126 sn = reductionState.number;
1127 tn = reductionState.token;
1128 }
1129 else if (processState == syntaxError) {
1130 tn = 0;
1131 }
1132 tuple_dict *d = xis(sn);
1133 int *items = d->text;
1134 int nitems = d->nsx;
1135 items += 2*nitems;
1136 LOGV(state.number);
1137 LOGV(state.token);
1138 LOGV(stateStack.size());
1139 LOGV(processState);
1140 LOGV(ntkns);
1141 sx = stateStack.size();
1142 if (processState == selectionRequired) {
1143 int rx = Rule(ruleToReduce)->length();
1144 int n = stateStack.size();
1145 for (sx = reductionIndex; rx >= 0; rx--) {
1146 int index = sx + rx;
1147 if (index > n) {
1148 continue;
1149 }
1150 if (index < n) {
1151 sn = stateStack[index].number;
1152 }
1153 at(isl, index, sn, ruleToReduce, rx);
1154 }
1155 tuple_dict *d = xis(reductionState.number);
1156 int *items = d->text;
1157 int nitems = d->nsx;
1158 items += 2*nitems;
1159 while (nitems--) {
1160 fx = *--items;
1161 fn = *--items;
1162 LOGV(fn);
1163 LOGV(fx);
1164 if ((unsigned) fx >= Rule(fn)->length()) {
1165 continue;
1166 }
1167 if (x3a(isl, sx, fn, fx)) {
1168 at(isl, sx, sn, fn, fx);
1169 }
1170 }
1171 }
1172 else {
1173 state_number_map *sp = &map_state_number[sn];
1174 unsigned *tokenPointer = lstptr(*sp, t_actions);
1175
1176 unsigned k = 0;
1177 while (tokenPointer[k] &&
1178 tokenPointer[k] != (unsigned) tn && tokenPointer[k]) {
1179 k++;
1180 }
1181 if (k == 0 && lstptr(*sp, a_actions)[k] == pe_syn_error) {
1182 tn = 0;
1183 }
1184 LOGV(k);
1185 LOGV(tn);
1186 LOGV(nitems);
1187 if (tn.isNotNull()) {
1188 while (nitems--) {
1189 fx = *--items;
1190 fn = *--items;
1191 LOGV(fn);
1192 LOGV(fx);
1193 //if (x4(tn, fn)) {
1194 if (tn.isExpansionRule(fn)) {
1195 at(isl, sx, sn, fn, fx);
1196 continue;
1197 }
1198 Rule rule = fn;
1199 if ((unsigned) fx >= rule->length()) {
1200 continue;
1201 }
1202 //if (lstptr(map_form_number[fn],tokens)[fx] == tn) {
1203 // at(isl, sx, sn, fn, fx);
1204 //}
1205 if (rule.token(fx) == tn) {
1206 at(isl, sx, sn, (int) rule, fx);
1207 }
1208 }
1209 LOGV(isl->nt);
1210 for (k = 0; k < sp->n_completed_forms; k++) {
1211 unsigned rule = lstptr(*sp,completed_forms)[k];
1212 LOGV(rule);
1213 at(isl, sx, sn, rule, Rule(rule)->length());
1214 }
1215 LOGV(isl->nt);
1216 if (isl->nt) {
1217 items = d->text;
1218 nitems = d->nsx;
1219 items += 2*nitems;
1220 while (nitems--) {
1221 fx = *--items;
1222 fn = *--items;
1223 LOGV(fn);
1224 LOGV(fx);
1225 if ((unsigned) fx >= Rule(fn)->length()) {
1226 continue;
1227 }
1228 if (x3a(isl, sx, fn, fx)) {
1229 at(isl, sx, sn, fn, fx);
1230 }
1231 }
1232 }
1233 else {
1234 items = d->text;
1235 nitems = d->nsx;
1236 items += 2*nitems;
1237 while (nitems--) {
1238 fx = *--items;
1239 fn = *--items;
1240 LOGV(fn);
1241 LOGV(fx);
1242 at(isl, sx, sn, fn, fx);
1243 }
1244 }
1245 }
1246 else while (nitems--) {
1247 fx = *--items;
1248 fn = *--items;
1249 LOGV(fn);
1250 LOGV(fx);
1251 at(isl, sx, sn, fn, fx);
1252 }
1253 }
1254 LOGV(isl->nt);
1255 LOGV((int) d);
1256 delete_tuple_dict(d);
1257 LOGV(sx);
1258 while (sx-- > 0) {
1259 tuple_dict *d;
1260 tn = stateStack[sx].token;
1261 sn = stateStack[sx].number;
1262 LOGV(sn);
1263 LOGV(tn);
1264 d = xis(sn);
1265 items = d->text;
1266 nitems = d->nsx;
1267 items += 2*nitems;
1268 while (nitems--) {
1269 fx = *--items;
1270 fn = *--items;
1271 LOGV(fn);
1272 LOGV(fx);
1273
1274 if ((unsigned) fx >= Rule(fn)->length()) {
1275 continue;
1276 }
1277 if (x3(isl, sx, fn, fx)) {
1278 at(isl, sx, sn, fn, fx);
1279 }
1280 }
1281 delete_tuple_dict(d);
1282 }
1283 LOGV(isl->nt);
1284 return isl;
1285 }
1286