--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/anagram/agcore/ftpar.cpp	Sat Dec 22 17:52:45 2007 -0500
@@ -0,0 +1,1286 @@
+/*
+ * AnaGram, A System for Syntax Directed Programming
+ * Copyright 1993-2002 Parsifal Software. All Rights Reserved.
+ * See the file COPYING for license and usage terms.
+ *
+ * ftpar.cpp
+ */
+
+#include "agarray.h"
+#include "arrays.h"
+#include "bpe3.h"
+#include "cd.h"
+#include "config.h"
+#include "csexp.h"
+#include "dict.h"
+#include "ftpar.h"
+#include "keyword.h"
+#include "q1glbl.h"
+#include "rule.h"
+#include "token.h"
+#include "tsd.h"
+
+//#define INCLUDE_LOGGING
+#include "log.h"
+
+
+AgArray<unsigned> traceCounts;
+
+//static dc_ref trace_count_display;
+
+
+int precedes(cint a, cint b) {
+  if (a.y < b.y) return 1;
+  if (a.y > b.y) return 0;
+  if (a.x < b.x) return 1;
+  return 0;
+}
+
+
+FtParser::FtParser(AgString t)
+  : text(t)
+  , state(t.pointer())
+  , initialStack(0)
+  , lookAhead(state.pointer)
+  , endPointer(state.pointer +
+	       (state.pointer != 0 ? strlen((const char *) state.pointer) : 0))
+  , inputToken(0)
+  , nNullShifts(0)
+  , processState(ready)
+  , ruleToReduce(0)
+  , reductionSelection(0)
+  , stackChanged(*this)
+  , testingKeyword(0)
+{
+  LOGSECTION("FtParser::FtParser");
+  LOGV((int) state.pointer);
+  LOGV((int) lookAhead);
+  getToken();
+  if (!traceCounts.exists()) {
+    traceCounts = AgArray<unsigned>(nforms_base + 1);
+    memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
+/*
+    trace_count_display =
+      dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
+*/
+  }
+}
+
+FtParser::FtParser()
+  : initialStack(0)
+  , lookAhead(0)
+  , inputToken(0)
+  , nNullShifts(0)
+  , processState(ready)
+  , ruleToReduce(0)
+  , reductionSelection(0)
+  , stackChanged(*this)
+  , testingKeyword(0)
+{
+  LOGSECTION("FtParser::FtParser");
+  LOGV((int) state.pointer);
+  LOGV((int) lookAhead);
+  if (!traceCounts.exists()) {
+    traceCounts = AgArray<unsigned>(nforms_base + 1);
+    memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
+/*
+    trace_count_display =
+      dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
+*/
+  }
+}
+
+FtParser::FtParser(tsd *initialStack_)
+  : initialStack(initialStack_ ? copy_tuple_set(initialStack_) : 0)
+  , lookAhead(0)
+  , inputToken(0)
+  , nNullShifts(0)
+  , processState(ready)
+  , ruleToReduce(0)
+  , reductionSelection(0)
+  , stackChanged(*this)
+  , testingKeyword(0)
+{
+  LOGSECTION("FtParser::FtParser");
+  LOGV((int) state.pointer);
+  LOGV((int) lookAhead);
+  LOGV((int) initialStack);
+  if (initialStack) {
+    LOGV(initialStack->nt);
+    for (unsigned i = 0; i < initialStack->nt; i++) {
+      unsigned sn, tn;
+      xtx(initialStack,i, &sn, &tn);
+      stateStack.push(State(sn,tn));
+    }
+    state = stateStack.pop();
+  }
+  LOGV(stateStack.size());
+  if (!traceCounts.exists()) {
+    traceCounts = AgArray<unsigned>(nforms_base + 1);
+    memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
+/*
+    trace_count_display =
+      dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
+*/
+  }
+}
+
+FtParser::~FtParser() {
+  if (initialStack) {
+    delete_tsd(initialStack);
+  }
+}
+
+FtParser &FtParser::reset() {
+  LOGSECTION("FtParser::reset");
+  stateStack.discardData();
+  auxStack.discardData();
+  transStack.discardData();
+  LOGV(auxStack.size());
+  LOGV((int) initialStack);
+  inputToken = 0;
+  if (initialStack) {
+    LOGV(initialStack->nt);
+    for (unsigned i = 0; i < initialStack->nt; i++) {
+      unsigned sn, tn;
+      xtx(initialStack,i, &sn, &tn);
+      stateStack.push(State(sn, tn));
+    }
+    state = stateStack.pop();
+  }
+  else {
+    state = State(text.pointer());
+    lookAhead = state.pointer;
+    endPointer = state.pointer +
+      (state.pointer != 0 ? strlen((const char *) state.pointer) : 0);
+    getToken();
+  }
+  LOGV(stateStack.size());
+  reductionState = State();
+  nNullShifts = 0;
+  processState = ready;
+  //if ((dc *)displayControl) {
+  //  displayControl->des->d_size.y = stateStack.size();
+  //}
+  stackChanged(stateStack.size() + 1);
+  return *this;
+}
+
+void FtParser::track(void) {
+  LOGSECTION("FtParser::track");
+  LOGV((int) state.pointer);
+  LOGV((int) lookAhead);
+  if (lookAhead != 0) {
+    while (state.pointer < lookAhead) {
+      switch (*state.pointer++) {
+	case '\n':
+	  if (*state.pointer) state.column = state.charPos = 0, state.line++;
+	case '\f':
+	  break;
+	case '\t':
+	  state.column += tab_spacing - state.column % tab_spacing;
+	  state.charPos++;
+	  break;
+	default:
+	  state.column++;
+	  state.charPos++;
+      }
+    }
+  }
+  state.token = inputToken = 0;
+  auxStack.discardData();
+  transStack.discardData();
+  LOGV(auxStack.size());
+  nNullShifts = 0;
+  LOGV(state.line) LCV(state.column);
+}
+
+void FtParser::shiftTerminalAndAccept(void) {
+  LOGSECTION("FtParser::shiftTerminalAndAccept");
+  processState = finished;
+  track();
+}
+
+void FtParser::shiftTerminal() {
+  LOGSECTION("FtParser::shiftTerminal");
+  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
+  LOGV(stateStack.size());
+  state.token = map_state_number[actionParameter].char_token;
+  stateStack.push(state);
+  state.number = actionParameter;
+  track();
+  LOGV(state.token);
+  LOGV(state.number) LCV((int) state.pointer);
+  LOGV(stateStack.size());
+}
+
+void FtParser::requestSelection(int actionParameter) {
+  LOGSECTION("FtParser::requestSelection");
+  processState = selectionRequired;
+  ruleToReduce = actionParameter;
+  reductionSelection = 0;
+  while (!validSelection(reductionSelection, reductionState.number)) {
+    reductionSelection++;
+  }
+  LOGV(ruleToReduce);
+  LOGV(reductionIndex);
+  LOGV(state.number) LCV((int) state.pointer);
+  LOGV(reductionState.number);
+  reductionState.token = ibnfs[ibnfb[ruleToReduce]];
+  LOGV(stateStack.size());
+}
+
+/*
+ * State stack discipline
+ *   reduce
+ *     If n is the length of the rule, n states are popped from the
+ *     state stack.
+ *     If n > 0, the state number becomes the state number of the
+ *     last state popped.
+ *
+ *   shiftTerminalAndReduce
+ *     If n is the length of the rule, n-1 states are popped from
+ *     the state stack.
+ *     If n > 1, the state number becomes the state number of the
+ *     last state popped.
+ *
+ *   shiftNonterminalAndReduce
+ *     If n is the length of the rule, n-1 states are popped from
+ *     the state stack.
+ *     If n > 1, the state number becomes the state number of the
+ *     last state popped.
+ *
+ *   shiftNull
+ *     The current state is pushed to the state stack and the
+ *     new state number is given by the action parameter.
+ *
+ *   shiftNonterminal
+ *     The current state is pushed to the state stack and the
+ *     new state number is given by the action parameter
+ */
+
+void FtParser::shiftTerminalAndReduce() {
+  LOGSECTION("FtParser::shiftTerminalAndReduce");
+  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
+  LOGV(location());
+  LOGV(actionParameter);
+  //form_number_map *fp = &map_form_number[actionParameter];
+  Rule rule(actionParameter);
+  RuleDescriptor &ruleDescriptor(rule);
+  //ruleLength = rule->length();
+  ruleLength = ruleDescriptor.length();
+  LOGV(ruleLength);
+  if (actionParameter <= nforms_base) {
+    traceCounts[actionParameter]++;
+  }
+  int nStackedStates = ruleLength - 1;
+  if (nStackedStates > 0) {
+    reductionState = stateStack[stateStack.size() - nStackedStates];
+  }
+  else {
+    reductionState = state;
+  }
+  track();
+  if (ibnfn[actionParameter] > 1) {
+    reductionIndex = stateStack.size() - (ruleLength - 1);
+    assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
+    requestSelection(actionParameter);
+    return;
+  }
+  assert((unsigned)nStackedStates <= stateStack.size());
+  stateStack.discardData(nStackedStates);
+  //reductionState.token = rule->prim_tkn;
+  reductionState.token = ruleDescriptor.prim_tkn;
+  state.number = reductionState.number;
+  dispatchReductionToken();
+  LOGV(location());
+  LOGV(state.number);
+  LOGV(state.token) LCV((int) state.pointer);
+  auxStack.discardData();
+  transStack.discardData();
+  LOGV(auxStack.size());
+}
+
+void FtParser::reduce(void) {
+  LOGSECTION("FtParser::reduce");
+  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
+  //form_number_map *fp = &map_form_number[actionParameter];
+  Rule rule(actionParameter);
+  RuleDescriptor &ruleDescriptor(rule);
+  //ruleLength = rule->length();
+  ruleLength = ruleDescriptor.length();
+  LOGV(actionParameter);
+  LOGV(ruleLength);
+  LOGV(stateStack.size());
+  if (actionParameter <= nforms_base) {
+    traceCounts[actionParameter]++;
+  }
+  if (ruleLength) {
+    reductionState = stateStack[stateStack.size() - ruleLength];
+  }
+  else {
+    reductionState = state;
+  }
+  if (ibnfn[actionParameter] > 1) {
+    reductionIndex = stateStack.size() - ruleLength;
+    assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
+    requestSelection(actionParameter);
+    return;
+  }
+  int k = ruleLength;
+  transStack.push(Transaction(k, state));
+  LOGV(transStack.size());
+  LOGV(k) LCV(state.number) LCV(state.token);
+  LOGV(nNullShifts);
+  while (k--) {
+    LOGV(stateStack.top().number) LCV(stateStack.top().token) 
+      LCV((int) state.pointer);
+    auxStack.push(stateStack.pop());
+  }
+  LOGV(auxStack.size());
+  //reductionState.token = rule->prim_tkn;
+  reductionState.token = ruleDescriptor.prim_tkn;
+  state.number = reductionState.number;
+  LOGV(reductionState.token);
+  LOGV(stateStack.size());
+  dispatchReductionToken();
+  LOGV(stateStack.size());
+  if (lookAhead) {
+    lookAhead = state.pointer;
+  }
+  state.token = 0;
+}
+
+void FtParser::skip(void) {
+  LOGSECTION("FtParser::skip");
+  LOGV(state.number) LCV(state.token);
+  if (actionParameter <= nforms_base) {
+    traceCounts[actionParameter]++;
+  }
+  track();
+}
+
+void FtParser::shiftNull(void) {
+  LOGSECTION("FtParser::shiftNull");
+  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
+  transStack.push(Transaction(-1, state));
+  LOGV(transStack.size());
+  state.token = map_state_number[actionParameter].char_token;
+  stateStack.push(state);
+  state.number = actionParameter;
+  if (lookAhead) {
+    lookAhead = state.pointer;
+  }
+  state.token = 0;
+}
+
+void FtParser::error() {
+  LOGSECTION("FtParser::error");
+  LOGV(stateStack.size());
+  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
+  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
+  LOGV(auxStack.size());
+  int k = transStack.size();
+  LOGV(transStack.size());
+  while (k--) {
+    Transaction trans = transStack.pop();
+    int n = trans.count;
+    LOGV(trans.count) LCV(trans.state.number) LCV(trans.state.token);
+    while (n < 0) {
+      stateStack.pop();
+      n++;
+    }
+    while (n > 0) {
+      stateStack.push(auxStack.pop());
+      n--;
+    }
+    state.number = trans.state.number;
+    state.token = trans.state.token;
+    LOGV(trans.count) LCV(state.number) LCV(state.token);
+  }
+  auxStack.discardData();
+  transStack.discardData();
+  processState = syntaxError;
+  if (lookAhead) {
+    lookAhead = state.pointer;
+  }
+  state.token = 0;
+}
+
+
+void FtParser::accept(void) {
+  LOGSECTION("FtParser::accept");
+  state.number = stateStack.top().number;
+  state.token = stateStack.top().token;
+  stateStack.pop();
+  LOGV(stateStack.size());
+  processState = finished;
+}
+
+int FtParser::shiftNonterminal() {
+  LOGSECTION("FtParser::shiftNonterminal");
+  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
+  transStack.push(Transaction(-1, state));
+  LOGV(transStack.size());
+  LOGV(actionParameter) LCV(nstates);
+  reductionState.token = map_state_number[actionParameter].char_token;
+#ifdef INCLUDE_LOGGING
+  ics();
+  atkn(reductionState.token);
+  LOGV(reductionState.token) LCV(string_base);
+  rcs();
+#endif
+  stateStack.push(reductionState);
+  state.number = actionParameter;
+  LOGV(stateStack.size());
+  LOGV(state.number) LCV((int) state.pointer);
+  return 0;
+}
+
+int FtParser::shiftNonterminalAndReduce() {
+  LOGSECTION("FtParser::shiftNonterminalAndReduce");
+  LOGV(location());
+  LOGV(reductionState.number) LCV(reductionState.token);
+  //form_number_map *fp = &map_form_number[actionParameter];
+  Rule rule(actionParameter);
+  RuleDescriptor &ruleDescriptor(rule);
+  //ruleLength = rule->length();
+  ruleLength = ruleDescriptor.length();
+  LOGS("rule number") LCV(actionParameter);
+  LOGV(ruleLength);
+  LOGV(rule->prim_tkn);
+  LOGV(stateStack.size());
+  if (actionParameter <= nforms_base) {
+    traceCounts[actionParameter]++;
+  }
+  int nStackedStates = ruleLength - 1;
+  if (nStackedStates > 0) {
+    reductionState = stateStack[stateStack.size() - nStackedStates];
+  }
+  if (ibnfn[actionParameter] > 1) {
+    reductionIndex = stateStack.size() - (ruleLength - 1);
+    assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
+    requestSelection(actionParameter);
+    return 0;
+  }
+  transStack.push(Transaction(nStackedStates, state));
+  LOGV(transStack.size());
+  LOGV(nStackedStates) LCV(state.number) LCV(state.token);
+  LOGV(auxStack.size());
+  while (nStackedStates--) {
+    LOGV(stateStack.top().number) LCV(stateStack.top().token) 
+      LCV((int) state.pointer);
+    auxStack.push(stateStack.pop());
+  }
+  LOGV(auxStack.size());
+  //reductionState.token = rule->prim_tkn;
+  reductionState.token = ruleDescriptor.prim_tkn;
+  state.number = reductionState.number;
+  LOGV(location());
+  LOGV(stateStack.size());
+  return 1;
+}
+
+int FtParser::shiftNonterminalAndAccept() {
+  LOGSECTION("FtParser::shiftNonTerminalAndAccept");
+  state.number = reductionState.number;
+  state.token = reductionState.token;
+  LOGV(state.number);
+  LOGV(state.token) LCV((int) state.pointer);
+  processState = finished;
+  return 0;
+}
+
+int (FtParser::*FtParser::nonterminalAction[4])() = {
+  &FtParser::shiftNonterminalAndAccept,
+  &FtParser::shiftNonterminal,
+  &FtParser::shiftNonterminalAndReduce,
+  &FtParser::shiftNonterminalAndReduce
+};
+
+void (FtParser::*FtParser::terminalAction[11])() = {
+  &FtParser::shiftTerminalAndAccept,
+  &FtParser::shiftTerminal,
+  &FtParser::shiftTerminalAndReduce,
+  &FtParser::shiftTerminalAndReduce,
+  &FtParser::reduce,
+  &FtParser::reduce,
+  &FtParser::accept,
+  &FtParser::error,
+  &FtParser::shiftNull,
+  &FtParser::skip,
+  &FtParser::skip
+};
+
+static int different(const unsigned char *s1, const unsigned char *s2, unsigned n) {
+  LOGSECTION("different");
+  LOGV(n);
+  //if (n > strlen((const char *)s2)) return 1;
+  if (case_sensitive) {
+    return strncmp((const char *)s1, (const char *) s2, n);
+  }
+  while (n--) {
+    if (agToUpper(*s1++) != agToUpper(*s2++)) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+
+Token FtParser::keyToken(void) {
+  LOGSECTION("FtParser::keyToken");
+  int matchLength = 0;
+  //int key = 0;
+  Keyword key;
+  int keyListNumber = map_state_number[state.number].key_list;
+  if (keyListNumber == 0) {
+    return Token();
+  }
+
+  int *keyList = dict_str(key_list_dict, keyListNumber);
+  int nKeys = *keyList++ - 1;
+  LOGV(lookAhead) LCV(nKeys);
+  while (nKeys--) {
+    Keyword keyNumber = map_token_number[*keyList++].key;
+    KeywordDescriptor &keyDescriptor(keyNumber);
+    //unsigned char *keyString = (unsigned char *)keyNumber->string.pointer();
+    unsigned char *keyString = (unsigned char *)keyDescriptor.string.pointer();
+    //AgString keyString = keyNumber->string;
+    int length = strlen((const char *) keyString);
+    //int length = keyString.size();
+    if (length <= matchLength) {
+      continue;
+    }
+    if ((lookAhead + length) > endPointer) {
+      continue;
+    }
+    if (different(keyString, lookAhead, length)) {
+      continue;
+    }
+    //if (different((unsigned char *)keyString.pointer(), lookAhead, length)) {
+    //  continue;
+    //}
+    //int charSetNumber = keyNumber->reserve;
+    int charSetNumber = keyDescriptor.reserve;
+    if (charSetNumber) {
+      int *reservedCharSet = dict_str(char_set_dict, charSetNumber);
+      int nReservedCharSet = *reservedCharSet++ - 1;
+      if (lookAhead + length < endPointer) {
+        unsigned char fc = state.pointer[length];
+        while (nReservedCharSet && fc != *reservedCharSet++) {
+	  nReservedCharSet--;
+	}
+        if (nReservedCharSet) {
+	  continue;
+	}
+      }
+    }
+    matchLength = length;
+    key = keyNumber;
+  }
+  LOGV((int) lookAhead) LCV(matchLength);
+  LOGV(key) LCV(testingKeyword);
+  if (key.isNotNull() && (int) key != testingKeyword) {
+    lookAhead = state.pointer + matchLength;
+    LOGV(key->token_number);
+    return key->token_number;
+  }
+  return Token();
+}
+
+
+void FtParser::getToken() {
+  LOGSECTION("FtParser::getToken");
+  LOGV(stateStack.size());
+  LOGV(state.number) LCV((int) state.pointer);
+  LOGV((int) lookAhead);
+  LOGV((int) endPointer);
+  LOGV(inputToken);
+  if (inputToken) {
+    state.token = inputToken;
+  }
+  else if (lookAhead != 0 && lookAhead >= endPointer) {
+    if (text.exists() && eof_token) {
+      token_number_map &eofTokenMap = map_token_number[eof_token];
+      //if (map_token_number[eof_token].non_terminal_flag) {
+      if (eofTokenMap.non_terminal_flag) {
+        //unsigned charSet = map_token_number[eof_token].token_set_id;
+        unsigned charSet = eofTokenMap.token_set_id;
+        unsigned *list = (unsigned *) dict_str(char_set_dict,charSet);
+        int character = list[1];
+        LOGV(charSet);
+        LOGV(character);
+        state.token = map_char_number[character].token_number;
+      }
+      else {
+	state.token = eof_token;
+      }
+      LOGV(state.token) LCV((int) state.pointer);
+    }
+    else {
+      processState = unexpectedEndOfFile;
+      state.token = 0;
+    }
+  }
+  else if (lookAhead != 0) {
+    state.token = keyToken();
+    LOGV(state.token) LCV((int) state.pointer);
+    if (state.token == 0) {
+      state.token = 
+	map_char_number[*lookAhead++ - min_char_number].token_number;
+    }
+  }
+  LOGV(state.token) LCV((int) state.pointer);
+  LOGV(auxStack.size());
+}
+
+/*
+unsigned FtParser::inspectToken() {
+  LOGSECTION("FtParser::inspectToken");
+  LOGV((int) lookAhead);
+  LOGV((int) endPointer);
+  unsigned token = 0;
+  if (inputToken) {
+    token = inputToken;
+  }
+  else if (lookAhead !=0 && lookAhead >= endPointer) {
+    if (text.exists() && eof_token) {
+      if (map_token_number[eof_token].non_terminal_flag) {
+        unsigned charSet = map_token_number[eof_token].token_set_id;
+        unsigned *list = (unsigned *) dict_str(char_set_dict,charSet);
+        int character = list[1];
+        LOGV(charSet);
+        LOGV(character);
+        token = map_char_number[character].token_number;
+      }
+      else {
+        token = eof_token;
+      }
+      LOGV(token);
+    }
+    else {
+      token = 0;
+    }
+  }
+  else if (lookAhead != 0) {
+    const unsigned char *save = lookAhead;
+    token = keyToken();
+    lookAhead = save;
+    LOGV(token);
+    if (token == 0) {
+      token = map_char_number[*lookAhead - min_char_number].token_number;
+    }
+  }
+  LOGV(token);
+  LOGV(auxStack.size());
+  return token;
+}
+*/
+
+void FtParser::dispatchReductionToken() {
+  LOGSECTION("FtParser::dispatchReductionToken");
+  unsigned k;
+  state_number_map *sp;
+
+  do {
+    LOGV(reductionState.number);
+    LOGV(reductionState.token);
+    sp = &map_state_number[reductionState.number];
+    unsigned *tokenPointer = lstptr(*sp,t_actions);
+    for (k = sp->n_actions; k && tokenPointer[--k] != reductionState.token;) {
+      /* nothing */
+    }
+    LOGV(k) LCV(tokenPointer[k]);
+    assert(tokenPointer[k] == reductionState.token);
+    actionParameter = lstptr(*sp,p_actions)[k];
+    LOGV(k) LCV(actionParameter);
+    if (k == 0) {
+      shiftNonterminal();
+      processState = selectionError;
+      stackChanged(stateStack.size()+1);
+      return;
+    }
+  } while ((this->*(nonterminalAction[lstptr(*sp,a_actions)[k]]))());
+  stackChanged(stateStack.size()+1);
+}
+
+void FtParser::completeReduction(int token) {
+  LOGSECTION("FtParser::completeReduction(int)");
+  int k = stateStack.size() - reductionIndex;
+  assert((unsigned) k <= (unsigned) stateStack.size());
+  LOGV(k);
+  stateStack.discardData(k);
+  reductionState.token = token;
+  LOGV(token);
+  processState = running;
+  dispatchReductionToken();
+  if (processState == running &&
+      state.pointer == lookAhead &&
+      state.token == 0) {
+    getToken();
+  }
+  nNullShifts = 0;
+  auxStack.discardData();
+  transStack.discardData();
+  LOGV(auxStack.size());
+}
+
+void FtParser::completeReduction() {
+  LOGSECTION("FtParser::completeReduction()");
+  int token = ibnfs[ibnfb[ruleToReduce]+reductionSelection];
+  completeReduction(token);
+}
+
+void FtParser::parseAction() {
+  LOGSECTION("FtParser::parseAction");
+  state_number_map *sp = &map_state_number[state.number];
+  unsigned *tokenPointer = lstptr(*sp, t_actions);
+
+  if (processState < running) {
+    processState = running;
+  }
+  int k = 0;
+  while (tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
+    k++;
+  }
+
+  actionParameter = lstptr(*sp, p_actions)[k];
+  ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
+
+  if (processState <= running && state.token == 0) {
+    getToken();
+  }
+  stackChanged(stateStack.size()+1);
+}
+
+void FtParser::stepToken(unsigned token) {
+  LOGSECTION("FtParser::stepToken");
+  LOGV(state.number);
+  LOGV(state.token) LCV((int) state.pointer);
+  lookAhead = state.pointer = 0;
+  inputToken = token;
+  processState = running;
+
+  if (map_token_number[token].non_terminal_flag) {
+    reductionState = state;
+    reductionState.token = token;
+    state.token = inputToken = 0;
+    dispatchReductionToken();
+    transStack.discardData();
+    auxStack.discardData();
+    if (processState <= running && state.token == 0) {
+      getToken();
+    }
+    stackChanged(stateStack.size());
+    return;
+  }
+  inputToken = token;
+  state_number_map *sp = &map_state_number[state.number];
+  unsigned *tokenPointer = lstptr(*sp, t_actions);
+  state.token = token;
+
+  int k = 0;
+  while(tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
+    k++;
+  }
+
+  actionParameter = lstptr(*sp, p_actions)[k];
+  ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
+  LOGV(processState) LCV(state.token);
+  if (processState <= running && state.token == 0) {
+    getToken();
+  }
+  stackChanged(stateStack.size()+1);
+}
+
+void FtParser::parseToken(unsigned token) {
+  LOGSECTION("FtParser::parseToken");
+  LOGV(state.number) LCV(token);
+  LOGV(state.token) LCV((int) state.pointer);
+  lookAhead = state.pointer = 0;
+  inputToken = token;
+  processState = token ? running : syntaxError;
+  if (processState == syntaxError) {
+    return;
+  }
+
+  if (map_token_number[token].non_terminal_flag) {
+    reductionState = state;
+    reductionState.token = token;
+    state.token = inputToken = 0;
+    dispatchReductionToken();
+    if (processState <= running && state.token == 0) {
+      getToken();
+    }
+    return;
+  }
+  inputToken = state.token = token;
+
+  while (processState <= running && inputToken != 0) {
+    state_number_map *sp = &map_state_number[state.number];
+    unsigned *tokenPointer = lstptr(*sp, t_actions);
+    int k = 0;
+    while(tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
+      k++;
+    }
+    LOGV(k) LCV(sp->n_actions);
+    assert((unsigned) k < sp->n_actions);
+    actionParameter = lstptr(*sp, p_actions)[k];
+    ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
+    LOGV(processState) LCV(state.token);
+    if (processState <= running && inputToken != 0) {
+      state.token = inputToken;
+    }
+  }
+  if (processState <= running && state.token == 0) {
+    getToken();
+  }
+  stackChanged(stateStack.size()+1);
+}
+
+
+
+int FtParser::validToken(unsigned token, unsigned sn) {
+  LOGSECTION("FtParser::validToken");
+  state_number_map *sp = &map_state_number[sn];
+  unsigned *tokenPointer = lstptr(*sp, t_actions);
+
+  int k = 0;
+  while (tokenPointer[k] && tokenPointer[k] != token && tokenPointer[k]) {
+    k++;
+  }
+  if (k == 0 && lstptr(*sp, a_actions)[k] == pe_syn_error) {
+    return 0;
+  }
+  LOGV(sn);
+  LOGV(token);
+  LOGV(k);
+  LOGV(tokenPointer[k]);
+  return 1;
+}
+
+int FtParser::validSelection(unsigned selection, unsigned sn) {
+  LOGSECTION("FtParser::validSelection");
+  state_number_map *sp = &map_state_number[sn];
+  unsigned *tokenPointer = lstptr(*sp, t_actions);
+
+  unsigned token = ibnfs[ibnfb[ruleToReduce] + selection];
+
+  int k;
+  for (k = sp->n_actions; k && tokenPointer[--k] != token; ) {
+    LOGV(k);
+    LOGV(tokenPointer[k]);
+  }
+  LOGV(sn);
+  LOGV(token);
+  LOGV(k);
+  LOGV(tokenPointer[k]);
+  return k != 0;
+}
+
+FtParser &FtParser::parseTo(unsigned char *target) {
+  LOGSECTION("FtParser::parseTo");
+  int backup = 0;
+  while (stateStack.size() && state.pointer <= target) {
+    stateStack.pop(state);
+    backup = 1;
+  }
+  if (backup) {
+    processState = running;
+    lookAhead = state.pointer;
+    auxStack.discardData();
+    transStack.discardData();
+    LOGV(auxStack.size());
+    nNullShifts = 0;
+    getToken();
+  }
+  else if (processState == finished) {
+    reset();
+  }
+  if (processState < running) {
+    processState = running;
+  }
+  LOGV(state.token) LCV((int) state.pointer);
+  while (processState <= running && state.pointer < target) {
+    parseAction();
+  }
+  //if ((dc *)displayControl) {
+  //  displayControl->des->d_size.y = stateStack.size()+1;
+  //}
+  stackChanged(stateStack.size()+1);
+  return *this;
+}
+
+FtParser &FtParser::parse() {
+  if (processState < running) {
+    processState = running;
+  }
+  while (processState <= running) {
+    parseAction();
+  }
+  //if ((dc *)displayControl) {
+  //  displayControl->des->d_size.y = stateStack.size()+1;
+  //}
+  stackChanged(stateStack.size()+1);
+  return *this;
+}
+
+FtParser &FtParser::parse(const char *fragment) {
+  LOGSECTION("FtParser::parse(const char *)");
+  LOGV(fragment);
+  lookAhead = state.pointer = (const unsigned char *) fragment;
+  inputToken = 0;
+  endPointer = (const unsigned char *) fragment + strlen(fragment);
+  getToken();
+  LOGV(processState) LCV(inputToken);
+
+  while (processState <= running) {
+    parseAction();
+    LOGV(state.pointer);
+  }
+  //if ((dc *)displayControl) {
+  //  displayControl->des->d_size.y = stateStack.size()+1;
+  //}
+  stackChanged(stateStack.size()+1);
+  if (processState == unexpectedEndOfFile) {
+    processState = ready;
+  }
+  return *this;
+}
+
+FtParser &FtParser::prime(const char *fragment) {
+  LOGSECTION("FtParser::parse(const char *)");
+  LOGV(fragment);
+  lookAhead = state.pointer = (const unsigned char *) fragment;
+  endPointer = (const unsigned char *) fragment + strlen(fragment);
+  getToken();
+  LOGV(processState);
+  stackChanged(stateStack.size()+1);
+  return *this;
+}
+
+FtParser &FtParser::parseTo(cint *loc) {
+  LOGSECTION("FtParser::parseTo");
+  LOGV(*loc);
+  LOGV(stateStack.size());
+  LOGV(location());
+  int backup = 0;
+  LOGV(processState);
+  while (stateStack.size()
+	 && (loc->y < state.line
+	     || (loc->y == state.line && loc->x <= state.charPos))) {
+    stateStack.pop(state);
+    backup = 1;
+  }
+  LOGV(*loc) LCV(location());
+  LOGV(precedes(*loc, location()));
+  if (backup) {
+    processState = running;
+    lookAhead = state.pointer;
+    getToken();
+  }
+  else if (precedes(*loc, location())) {
+    reset();
+  }
+  LOGV(backup);
+  LOGV(processState);
+  LOGV(stateStack.size());
+  LOGV(state.number) LCV((int) state.pointer);
+  LOGV(location());
+  if (processState < running) {
+    processState = running;
+  }
+  while (processState == running &&
+         (state.line < loc->y || (state.line == loc->y &&
+				  state.charPos < loc->x))) {
+    parseAction();
+  }
+  LOGV(location());
+  //if ((dc *)displayControl) {
+  //  displayControl->des->d_size.y = stateStack.size()+1;
+  //}
+  stackChanged(stateStack.size()+1);
+  return *this;
+}
+
+FtParser &FtParser::step() {
+  LOGSECTION("FtParser::step");
+  LOGV(location());
+  if (processState < running) {
+    processState = running;
+  }
+  if (processState == running) {
+    parseAction();
+  }
+  //if ((dc *)displayControl) {
+  //  displayControl->des->d_size.y = stateStack.size()+1;
+  //}
+  stackChanged(stateStack.size()+1);
+  return *this;
+}
+
+FtParser &FtParser::step(char *fragment) {
+  LOGSECTION("FtParser::step(const char *)");
+  LOGV(fragment);
+  lookAhead = state.pointer = (unsigned char *) fragment;
+  endPointer = (unsigned char *) fragment + strlen(fragment);
+  getToken();
+  LOGV(processState);
+  if (processState <= running) {
+    parseAction();
+  }
+  //if ((dc *)displayControl) {
+  //  displayControl->des->d_size.y = stateStack.size()+1;
+  //}
+  stackChanged(stateStack.size()+1);
+  if (processState <= running && state.token == 0) {
+    getToken();
+  }
+  if (processState == unexpectedEndOfFile) {
+    processState = ready;
+  }
+  return *this;
+}
+
+const char *FtParser::processStateText[] = {
+  "Ready",
+  "Ready",                                   //running,
+  "Parse complete",                          //finished,
+  "Syntax error",           //syntaxError,
+  "Unexpected end of file", //unexpectedEndOfFile,
+  "Select reduction token",  //selectionRequired
+  "Selection error"
+};
+
+#if 0 /* unused */
+tsd *x1x_new(pcb_type *pcb) {
+  int sx, sn, tn, /*fn,*/ fx;
+  int *items;
+  int nitems;
+  tsd *isl = init_tsd(4);
+
+  ok_ptr(pcb);
+  sn = PCB.s.sn;
+  tn = PCB.token_number;
+  if (PCB.exit_flag) {
+    tn = 0;
+  }
+  sx = PCB.ssx;
+  if (PCB.reduction_token) {
+    sx -= PCB.rule_length;
+  }
+
+  {
+    tuple_dict *d;
+    d = xis(sn);
+    items = d->text;
+    nitems = d->nsx;
+    items += 2*nitems;
+    while (nitems--) {
+      fx = *--items;
+      Rule rule = *--items;
+      if (tn == 0 ||
+          fx >= rule->non_vanishing_length ||
+          rule.token(fx).isExpansionToken(tn)) {
+	//x2(Rule(fn)->token(fx), tn))
+	//x2(lstptr(map_form_number[fn],tokens)[fx], tn))
+        at(isl,sx,sn,(int) rule,fx);
+      }
+    }
+    delete_tuple_dict(d);
+  }
+
+  while (sx-- > 0) {
+    tuple_dict *d;
+    tn = map_state_number[sn].char_token;
+    sn = PCB.ss[sx].sn;
+    d = xis(sn);
+    items = d->text;
+    nitems = d->nsx;
+    items += 2*nitems;
+    while (nitems--) {
+      fx = *--items;
+      Rule rule = *--items;
+      if (fx >= rule->length()) {
+	continue;
+      }
+      if (x3(isl, sx, (int)rule, fx)) {
+	at(isl, sx, sn, (int) rule, fx);
+      }
+    }
+    delete_tuple_dict(d);
+  }
+  return isl;
+}
+#endif /* 0 - unused */
+
+tsd *FtParser::x1x_new() {
+  LOGSECTION("FtParser::x1x_new");
+  int sx, sn, fn, fx;
+  tsd *isl = init_tsd(4);
+
+  sn = state.number;
+  Token tn = state.token;
+  if (processState == selectionRequired) {
+    sn = reductionState.number;
+    tn = reductionState.token;
+  }
+  else if (processState == syntaxError) {
+    tn = 0;
+  }
+  tuple_dict *d = xis(sn);
+  int *items = d->text;
+  int nitems = d->nsx;
+  items += 2*nitems;
+  LOGV(state.number);
+  LOGV(state.token);
+  LOGV(stateStack.size());
+  LOGV(processState);
+  LOGV(ntkns);
+  sx = stateStack.size();
+  if (processState == selectionRequired) {
+    int rx = Rule(ruleToReduce)->length();
+    int n = stateStack.size();
+    for (sx = reductionIndex; rx >= 0; rx--) {
+      int index = sx + rx;
+      if (index > n) {
+	continue;
+      }
+      if (index < n) {
+	sn = stateStack[index].number;
+      }
+      at(isl, index, sn, ruleToReduce, rx);
+    }
+    tuple_dict *d = xis(reductionState.number);
+    int *items = d->text;
+    int nitems = d->nsx;
+    items += 2*nitems;
+    while (nitems--) {
+      fx = *--items;
+      fn = *--items;
+      LOGV(fn);
+      LOGV(fx);
+      if ((unsigned) fx >= Rule(fn)->length()) {
+	continue;
+      }
+      if (x3a(isl, sx, fn, fx)) {
+	at(isl, sx, sn, fn, fx);
+      }
+    }
+  }
+  else {
+    state_number_map *sp = &map_state_number[sn];
+    unsigned *tokenPointer = lstptr(*sp, t_actions);
+
+    unsigned k = 0;
+    while (tokenPointer[k] && 
+	   tokenPointer[k] != (unsigned) tn && tokenPointer[k]) {
+      k++;
+    }
+    if (k == 0 && lstptr(*sp, a_actions)[k] == pe_syn_error) {
+      tn = 0;
+    }
+    LOGV(k);
+    LOGV(tn);
+    LOGV(nitems);
+    if (tn.isNotNull()) {
+      while (nitems--) {
+        fx = *--items;
+        fn = *--items;
+        LOGV(fn);
+        LOGV(fx);
+        //if (x4(tn, fn)) {
+        if (tn.isExpansionRule(fn)) {
+          at(isl, sx, sn, fn, fx);
+          continue;
+        }
+        Rule rule = fn;
+        if ((unsigned) fx >= rule->length()) {
+	  continue;
+	}
+        //if (lstptr(map_form_number[fn],tokens)[fx] == tn) {
+	//  at(isl, sx, sn, fn, fx);
+	//}
+        if (rule.token(fx) == tn) {
+	  at(isl, sx, sn, (int) rule, fx);
+	}
+      }
+      LOGV(isl->nt);
+      for (k = 0; k < sp->n_completed_forms; k++) {
+        unsigned rule = lstptr(*sp,completed_forms)[k];
+        LOGV(rule);
+        at(isl, sx, sn, rule, Rule(rule)->length());
+      }
+      LOGV(isl->nt);
+      if (isl->nt) {
+        items = d->text;
+        nitems = d->nsx;
+        items += 2*nitems;
+        while (nitems--) {
+          fx = *--items;
+          fn = *--items;
+          LOGV(fn);
+          LOGV(fx);
+          if ((unsigned) fx >= Rule(fn)->length()) {
+	    continue;
+	  }
+          if (x3a(isl, sx, fn, fx)) {
+	    at(isl, sx, sn, fn, fx);
+	  }
+        }
+      }
+      else {
+        items = d->text;
+        nitems = d->nsx;
+        items += 2*nitems;
+        while (nitems--) {
+          fx = *--items;
+          fn = *--items;
+          LOGV(fn);
+          LOGV(fx);
+          at(isl, sx, sn, fn, fx);
+        }
+      }
+    }
+    else while (nitems--) {
+      fx = *--items;
+      fn = *--items;
+      LOGV(fn);
+      LOGV(fx);
+      at(isl, sx, sn, fn, fx);
+    }
+  }
+  LOGV(isl->nt);
+  LOGV((int) d);
+  delete_tuple_dict(d);
+  LOGV(sx);
+  while (sx-- > 0) {
+    tuple_dict *d;
+    tn = stateStack[sx].token;
+    sn = stateStack[sx].number;
+    LOGV(sn);
+    LOGV(tn);
+    d = xis(sn);
+    items = d->text;
+    nitems = d->nsx;
+    items += 2*nitems;
+    while (nitems--) {
+      fx = *--items;
+      fn = *--items;
+      LOGV(fn);
+      LOGV(fx);
+
+      if ((unsigned) fx >= Rule(fn)->length()) {
+	continue;
+      }
+      if (x3(isl, sx, fn, fx)) {
+	at(isl, sx, sn, fn, fx);
+      }
+    }
+    delete_tuple_dict(d);
+  }
+  LOGV(isl->nt);
+  return isl;
+}
+