/*
 * AnaGram, A System for Syntax Directed Programming
 * Copyright 1993-2002 Parsifal Software. All Rights Reserved.
 * See the file COPYING for license and usage terms.
 *
 * ftpar.cpp
 */

#include "agarray.h"
#include "arrays.h"
#include "bpe3.h"
#include "cd.h"
#include "config.h"
#include "csexp.h"
#include "dict.h"
#include "ftpar.h"
#include "keyword.h"
#include "q1glbl.h"
#include "rule.h"
#include "token.h"
#include "tsd.h"

//#define INCLUDE_LOGGING
#include "log.h"


AgArray<unsigned> traceCounts;

//static dc_ref trace_count_display;


int precedes(cint a, cint b) {
  if (a.y < b.y) return 1;
  if (a.y > b.y) return 0;
  if (a.x < b.x) return 1;
  return 0;
}


FtParser::FtParser(AgString t)
  : text(t)
  , state(t.pointer())
  , initialStack(0)
  , lookAhead(state.pointer)
  , endPointer(state.pointer +
	       (state.pointer != 0 ? strlen((const char *) state.pointer) : 0))
  , inputToken(0)
  , nNullShifts(0)
  , processState(ready)
  , ruleToReduce(0)
  , reductionSelection(0)
  , stackChanged(*this)
  , testingKeyword(0)
{
  LOGSECTION("FtParser::FtParser");
  LOGV((int) state.pointer);
  LOGV((int) lookAhead);
  getToken();
  if (!traceCounts.exists()) {
    traceCounts = AgArray<unsigned>(nforms_base + 1);
    memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
/*
    trace_count_display =
      dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
*/
  }
}

FtParser::FtParser()
  : initialStack(0)
  , lookAhead(0)
  , inputToken(0)
  , nNullShifts(0)
  , processState(ready)
  , ruleToReduce(0)
  , reductionSelection(0)
  , stackChanged(*this)
  , testingKeyword(0)
{
  LOGSECTION("FtParser::FtParser");
  LOGV((int) state.pointer);
  LOGV((int) lookAhead);
  if (!traceCounts.exists()) {
    traceCounts = AgArray<unsigned>(nforms_base + 1);
    memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
/*
    trace_count_display =
      dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
*/
  }
}

FtParser::FtParser(tsd *initialStack_)
  : initialStack(initialStack_ ? copy_tuple_set(initialStack_) : 0)
  , lookAhead(0)
  , inputToken(0)
  , nNullShifts(0)
  , processState(ready)
  , ruleToReduce(0)
  , reductionSelection(0)
  , stackChanged(*this)
  , testingKeyword(0)
{
  LOGSECTION("FtParser::FtParser");
  LOGV((int) state.pointer);
  LOGV((int) lookAhead);
  LOGV((int) initialStack);
  if (initialStack) {
    LOGV(initialStack->nt);
    for (unsigned i = 0; i < initialStack->nt; i++) {
      unsigned sn, tn;
      xtx(initialStack,i, &sn, &tn);
      stateStack.push(State(sn,tn));
    }
    state = stateStack.pop();
  }
  LOGV(stateStack.size());
  if (!traceCounts.exists()) {
    traceCounts = AgArray<unsigned>(nforms_base + 1);
    memset(traceCounts.pointer(), 0, sizeof(unsigned)*traceCounts.size());
/*
    trace_count_display =
      dc_ref(new rule_count_dc("Trace Coverage", traceCounts));
*/
  }
}

FtParser::~FtParser() {
  if (initialStack) {
    delete_tsd(initialStack);
  }
}

FtParser &FtParser::reset() {
  LOGSECTION("FtParser::reset");
  stateStack.discardData();
  auxStack.discardData();
  transStack.discardData();
  LOGV(auxStack.size());
  LOGV((int) initialStack);
  inputToken = 0;
  if (initialStack) {
    LOGV(initialStack->nt);
    for (unsigned i = 0; i < initialStack->nt; i++) {
      unsigned sn, tn;
      xtx(initialStack,i, &sn, &tn);
      stateStack.push(State(sn, tn));
    }
    state = stateStack.pop();
  }
  else {
    state = State(text.pointer());
    lookAhead = state.pointer;
    endPointer = state.pointer +
      (state.pointer != 0 ? strlen((const char *) state.pointer) : 0);
    getToken();
  }
  LOGV(stateStack.size());
  reductionState = State();
  nNullShifts = 0;
  processState = ready;
  //if ((dc *)displayControl) {
  //  displayControl->des->d_size.y = stateStack.size();
  //}
  stackChanged(stateStack.size() + 1);
  return *this;
}

void FtParser::track(void) {
  LOGSECTION("FtParser::track");
  LOGV((int) state.pointer);
  LOGV((int) lookAhead);
  if (lookAhead != 0) {
    while (state.pointer < lookAhead) {
      switch (*state.pointer++) {
	case '\n':
	  if (*state.pointer) state.column = state.charPos = 0, state.line++;
	case '\f':
	  break;
	case '\t':
	  state.column += tab_spacing - state.column % tab_spacing;
	  state.charPos++;
	  break;
	default:
	  state.column++;
	  state.charPos++;
      }
    }
  }
  state.token = inputToken = 0;
  auxStack.discardData();
  transStack.discardData();
  LOGV(auxStack.size());
  nNullShifts = 0;
  LOGV(state.line) LCV(state.column);
}

void FtParser::shiftTerminalAndAccept(void) {
  LOGSECTION("FtParser::shiftTerminalAndAccept");
  processState = finished;
  track();
}

void FtParser::shiftTerminal() {
  LOGSECTION("FtParser::shiftTerminal");
  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
  LOGV(stateStack.size());
  state.token = map_state_number[actionParameter].char_token;
  stateStack.push(state);
  state.number = actionParameter;
  track();
  LOGV(state.token);
  LOGV(state.number) LCV((int) state.pointer);
  LOGV(stateStack.size());
}

void FtParser::requestSelection(int actionParameter) {
  LOGSECTION("FtParser::requestSelection");
  processState = selectionRequired;
  ruleToReduce = actionParameter;
  reductionSelection = 0;
  while (!validSelection(reductionSelection, reductionState.number)) {
    reductionSelection++;
  }
  LOGV(ruleToReduce);
  LOGV(reductionIndex);
  LOGV(state.number) LCV((int) state.pointer);
  LOGV(reductionState.number);
  reductionState.token = ibnfs[ibnfb[ruleToReduce]];
  LOGV(stateStack.size());
}

/*
 * State stack discipline
 *   reduce
 *     If n is the length of the rule, n states are popped from the
 *     state stack.
 *     If n > 0, the state number becomes the state number of the
 *     last state popped.
 *
 *   shiftTerminalAndReduce
 *     If n is the length of the rule, n-1 states are popped from
 *     the state stack.
 *     If n > 1, the state number becomes the state number of the
 *     last state popped.
 *
 *   shiftNonterminalAndReduce
 *     If n is the length of the rule, n-1 states are popped from
 *     the state stack.
 *     If n > 1, the state number becomes the state number of the
 *     last state popped.
 *
 *   shiftNull
 *     The current state is pushed to the state stack and the
 *     new state number is given by the action parameter.
 *
 *   shiftNonterminal
 *     The current state is pushed to the state stack and the
 *     new state number is given by the action parameter
 */

void FtParser::shiftTerminalAndReduce() {
  LOGSECTION("FtParser::shiftTerminalAndReduce");
  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
  LOGV(location());
  LOGV(actionParameter);
  //form_number_map *fp = &map_form_number[actionParameter];
  Rule rule(actionParameter);
  RuleDescriptor &ruleDescriptor(rule);
  //ruleLength = rule->length();
  ruleLength = ruleDescriptor.length();
  LOGV(ruleLength);
  if (actionParameter <= nforms_base) {
    traceCounts[actionParameter]++;
  }
  int nStackedStates = ruleLength - 1;
  if (nStackedStates > 0) {
    reductionState = stateStack[stateStack.size() - nStackedStates];
  }
  else {
    reductionState = state;
  }
  track();
  if (ibnfn[actionParameter] > 1) {
    reductionIndex = stateStack.size() - (ruleLength - 1);
    assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
    requestSelection(actionParameter);
    return;
  }
  assert((unsigned)nStackedStates <= stateStack.size());
  stateStack.discardData(nStackedStates);
  //reductionState.token = rule->prim_tkn;
  reductionState.token = ruleDescriptor.prim_tkn;
  state.number = reductionState.number;
  dispatchReductionToken();
  LOGV(location());
  LOGV(state.number);
  LOGV(state.token) LCV((int) state.pointer);
  auxStack.discardData();
  transStack.discardData();
  LOGV(auxStack.size());
}

void FtParser::reduce(void) {
  LOGSECTION("FtParser::reduce");
  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
  //form_number_map *fp = &map_form_number[actionParameter];
  Rule rule(actionParameter);
  RuleDescriptor &ruleDescriptor(rule);
  //ruleLength = rule->length();
  ruleLength = ruleDescriptor.length();
  LOGV(actionParameter);
  LOGV(ruleLength);
  LOGV(stateStack.size());
  if (actionParameter <= nforms_base) {
    traceCounts[actionParameter]++;
  }
  if (ruleLength) {
    reductionState = stateStack[stateStack.size() - ruleLength];
  }
  else {
    reductionState = state;
  }
  if (ibnfn[actionParameter] > 1) {
    reductionIndex = stateStack.size() - ruleLength;
    assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
    requestSelection(actionParameter);
    return;
  }
  int k = ruleLength;
  transStack.push(Transaction(k, state));
  LOGV(transStack.size());
  LOGV(k) LCV(state.number) LCV(state.token);
  LOGV(nNullShifts);
  while (k--) {
    LOGV(stateStack.top().number) LCV(stateStack.top().token) 
      LCV((int) state.pointer);
    auxStack.push(stateStack.pop());
  }
  LOGV(auxStack.size());
  //reductionState.token = rule->prim_tkn;
  reductionState.token = ruleDescriptor.prim_tkn;
  state.number = reductionState.number;
  LOGV(reductionState.token);
  LOGV(stateStack.size());
  dispatchReductionToken();
  LOGV(stateStack.size());
  if (lookAhead) {
    lookAhead = state.pointer;
  }
  state.token = 0;
}

void FtParser::skip(void) {
  LOGSECTION("FtParser::skip");
  LOGV(state.number) LCV(state.token);
  if (actionParameter <= nforms_base) {
    traceCounts[actionParameter]++;
  }
  track();
}

void FtParser::shiftNull(void) {
  LOGSECTION("FtParser::shiftNull");
  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
  transStack.push(Transaction(-1, state));
  LOGV(transStack.size());
  state.token = map_state_number[actionParameter].char_token;
  stateStack.push(state);
  state.number = actionParameter;
  if (lookAhead) {
    lookAhead = state.pointer;
  }
  state.token = 0;
}

void FtParser::error() {
  LOGSECTION("FtParser::error");
  LOGV(stateStack.size());
  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
  LOGV(auxStack.size());
  int k = transStack.size();
  LOGV(transStack.size());
  while (k--) {
    Transaction trans = transStack.pop();
    int n = trans.count;
    LOGV(trans.count) LCV(trans.state.number) LCV(trans.state.token);
    while (n < 0) {
      stateStack.pop();
      n++;
    }
    while (n > 0) {
      stateStack.push(auxStack.pop());
      n--;
    }
    state.number = trans.state.number;
    state.token = trans.state.token;
    LOGV(trans.count) LCV(state.number) LCV(state.token);
  }
  auxStack.discardData();
  transStack.discardData();
  processState = syntaxError;
  if (lookAhead) {
    lookAhead = state.pointer;
  }
  state.token = 0;
}


void FtParser::accept(void) {
  LOGSECTION("FtParser::accept");
  state.number = stateStack.top().number;
  state.token = stateStack.top().token;
  stateStack.pop();
  LOGV(stateStack.size());
  processState = finished;
}

int FtParser::shiftNonterminal() {
  LOGSECTION("FtParser::shiftNonterminal");
  LOGV(state.number) LCV(state.token) LCV((int) state.pointer);
  transStack.push(Transaction(-1, state));
  LOGV(transStack.size());
  LOGV(actionParameter) LCV(nstates);
  reductionState.token = map_state_number[actionParameter].char_token;
#ifdef INCLUDE_LOGGING
  ics();
  atkn(reductionState.token);
  LOGV(reductionState.token) LCV(string_base);
  rcs();
#endif
  stateStack.push(reductionState);
  state.number = actionParameter;
  LOGV(stateStack.size());
  LOGV(state.number) LCV((int) state.pointer);
  return 0;
}

int FtParser::shiftNonterminalAndReduce() {
  LOGSECTION("FtParser::shiftNonterminalAndReduce");
  LOGV(location());
  LOGV(reductionState.number) LCV(reductionState.token);
  //form_number_map *fp = &map_form_number[actionParameter];
  Rule rule(actionParameter);
  RuleDescriptor &ruleDescriptor(rule);
  //ruleLength = rule->length();
  ruleLength = ruleDescriptor.length();
  LOGS("rule number") LCV(actionParameter);
  LOGV(ruleLength);
  LOGV(rule->prim_tkn);
  LOGV(stateStack.size());
  if (actionParameter <= nforms_base) {
    traceCounts[actionParameter]++;
  }
  int nStackedStates = ruleLength - 1;
  if (nStackedStates > 0) {
    reductionState = stateStack[stateStack.size() - nStackedStates];
  }
  if (ibnfn[actionParameter] > 1) {
    reductionIndex = stateStack.size() - (ruleLength - 1);
    assert((unsigned) reductionIndex <= (unsigned) stateStack.size());
    requestSelection(actionParameter);
    return 0;
  }
  transStack.push(Transaction(nStackedStates, state));
  LOGV(transStack.size());
  LOGV(nStackedStates) LCV(state.number) LCV(state.token);
  LOGV(auxStack.size());
  while (nStackedStates--) {
    LOGV(stateStack.top().number) LCV(stateStack.top().token) 
      LCV((int) state.pointer);
    auxStack.push(stateStack.pop());
  }
  LOGV(auxStack.size());
  //reductionState.token = rule->prim_tkn;
  reductionState.token = ruleDescriptor.prim_tkn;
  state.number = reductionState.number;
  LOGV(location());
  LOGV(stateStack.size());
  return 1;
}

int FtParser::shiftNonterminalAndAccept() {
  LOGSECTION("FtParser::shiftNonTerminalAndAccept");
  state.number = reductionState.number;
  state.token = reductionState.token;
  LOGV(state.number);
  LOGV(state.token) LCV((int) state.pointer);
  processState = finished;
  return 0;
}

int (FtParser::*FtParser::nonterminalAction[4])() = {
  &FtParser::shiftNonterminalAndAccept,
  &FtParser::shiftNonterminal,
  &FtParser::shiftNonterminalAndReduce,
  &FtParser::shiftNonterminalAndReduce
};

void (FtParser::*FtParser::terminalAction[11])() = {
  &FtParser::shiftTerminalAndAccept,
  &FtParser::shiftTerminal,
  &FtParser::shiftTerminalAndReduce,
  &FtParser::shiftTerminalAndReduce,
  &FtParser::reduce,
  &FtParser::reduce,
  &FtParser::accept,
  &FtParser::error,
  &FtParser::shiftNull,
  &FtParser::skip,
  &FtParser::skip
};

static int different(const unsigned char *s1, const unsigned char *s2, unsigned n) {
  LOGSECTION("different");
  LOGV(n);
  //if (n > strlen((const char *)s2)) return 1;
  if (case_sensitive) {
    return strncmp((const char *)s1, (const char *) s2, n);
  }
  while (n--) {
    if (agToUpper(*s1++) != agToUpper(*s2++)) {
      return 1;
    }
  }
  return 0;
}


Token FtParser::keyToken(void) {
  LOGSECTION("FtParser::keyToken");
  int matchLength = 0;
  //int key = 0;
  Keyword key;
  int keyListNumber = map_state_number[state.number].key_list;
  if (keyListNumber == 0) {
    return Token();
  }

  int *keyList = dict_str(key_list_dict, keyListNumber);
  int nKeys = *keyList++ - 1;
  LOGV(lookAhead) LCV(nKeys);
  while (nKeys--) {
    Keyword keyNumber = map_token_number[*keyList++].key;
    KeywordDescriptor &keyDescriptor(keyNumber);
    //unsigned char *keyString = (unsigned char *)keyNumber->string.pointer();
    unsigned char *keyString = (unsigned char *)keyDescriptor.string.pointer();
    //AgString keyString = keyNumber->string;
    int length = strlen((const char *) keyString);
    //int length = keyString.size();
    if (length <= matchLength) {
      continue;
    }
    if ((lookAhead + length) > endPointer) {
      continue;
    }
    if (different(keyString, lookAhead, length)) {
      continue;
    }
    //if (different((unsigned char *)keyString.pointer(), lookAhead, length)) {
    //  continue;
    //}
    //int charSetNumber = keyNumber->reserve;
    int charSetNumber = keyDescriptor.reserve;
    if (charSetNumber) {
      int *reservedCharSet = dict_str(char_set_dict, charSetNumber);
      int nReservedCharSet = *reservedCharSet++ - 1;
      if (lookAhead + length < endPointer) {
        unsigned char fc = state.pointer[length];
        while (nReservedCharSet && fc != *reservedCharSet++) {
	  nReservedCharSet--;
	}
        if (nReservedCharSet) {
	  continue;
	}
      }
    }
    matchLength = length;
    key = keyNumber;
  }
  LOGV((int) lookAhead) LCV(matchLength);
  LOGV(key) LCV(testingKeyword);
  if (key.isNotNull() && (int) key != testingKeyword) {
    lookAhead = state.pointer + matchLength;
    LOGV(key->token_number);
    return key->token_number;
  }
  return Token();
}


void FtParser::getToken() {
  LOGSECTION("FtParser::getToken");
  LOGV(stateStack.size());
  LOGV(state.number) LCV((int) state.pointer);
  LOGV((int) lookAhead);
  LOGV((int) endPointer);
  LOGV(inputToken);
  if (inputToken) {
    state.token = inputToken;
  }
  else if (lookAhead != 0 && lookAhead >= endPointer) {
    if (text.exists() && eof_token) {
      token_number_map &eofTokenMap = map_token_number[eof_token];
      //if (map_token_number[eof_token].non_terminal_flag) {
      if (eofTokenMap.non_terminal_flag) {
        //unsigned charSet = map_token_number[eof_token].token_set_id;
        unsigned charSet = eofTokenMap.token_set_id;
        unsigned *list = (unsigned *) dict_str(char_set_dict,charSet);
        int character = list[1];
        LOGV(charSet);
        LOGV(character);
        state.token = map_char_number[character].token_number;
      }
      else {
	state.token = eof_token;
      }
      LOGV(state.token) LCV((int) state.pointer);
    }
    else {
      processState = unexpectedEndOfFile;
      state.token = 0;
    }
  }
  else if (lookAhead != 0) {
    state.token = keyToken();
    LOGV(state.token) LCV((int) state.pointer);
    if (state.token == 0) {
      state.token = 
	map_char_number[*lookAhead++ - min_char_number].token_number;
    }
  }
  LOGV(state.token) LCV((int) state.pointer);
  LOGV(auxStack.size());
}

/*
unsigned FtParser::inspectToken() {
  LOGSECTION("FtParser::inspectToken");
  LOGV((int) lookAhead);
  LOGV((int) endPointer);
  unsigned token = 0;
  if (inputToken) {
    token = inputToken;
  }
  else if (lookAhead !=0 && lookAhead >= endPointer) {
    if (text.exists() && eof_token) {
      if (map_token_number[eof_token].non_terminal_flag) {
        unsigned charSet = map_token_number[eof_token].token_set_id;
        unsigned *list = (unsigned *) dict_str(char_set_dict,charSet);
        int character = list[1];
        LOGV(charSet);
        LOGV(character);
        token = map_char_number[character].token_number;
      }
      else {
        token = eof_token;
      }
      LOGV(token);
    }
    else {
      token = 0;
    }
  }
  else if (lookAhead != 0) {
    const unsigned char *save = lookAhead;
    token = keyToken();
    lookAhead = save;
    LOGV(token);
    if (token == 0) {
      token = map_char_number[*lookAhead - min_char_number].token_number;
    }
  }
  LOGV(token);
  LOGV(auxStack.size());
  return token;
}
*/

void FtParser::dispatchReductionToken() {
  LOGSECTION("FtParser::dispatchReductionToken");
  unsigned k;
  state_number_map *sp;

  do {
    LOGV(reductionState.number);
    LOGV(reductionState.token);
    sp = &map_state_number[reductionState.number];
    unsigned *tokenPointer = lstptr(*sp,t_actions);
    for (k = sp->n_actions; k && tokenPointer[--k] != reductionState.token;) {
      /* nothing */
    }
    LOGV(k) LCV(tokenPointer[k]);
    assert(tokenPointer[k] == reductionState.token);
    actionParameter = lstptr(*sp,p_actions)[k];
    LOGV(k) LCV(actionParameter);
    if (k == 0) {
      shiftNonterminal();
      processState = selectionError;
      stackChanged(stateStack.size()+1);
      return;
    }
  } while ((this->*(nonterminalAction[lstptr(*sp,a_actions)[k]]))());
  stackChanged(stateStack.size()+1);
}

void FtParser::completeReduction(int token) {
  LOGSECTION("FtParser::completeReduction(int)");
  int k = stateStack.size() - reductionIndex;
  assert((unsigned) k <= (unsigned) stateStack.size());
  LOGV(k);
  stateStack.discardData(k);
  reductionState.token = token;
  LOGV(token);
  processState = running;
  dispatchReductionToken();
  if (processState == running &&
      state.pointer == lookAhead &&
      state.token == 0) {
    getToken();
  }
  nNullShifts = 0;
  auxStack.discardData();
  transStack.discardData();
  LOGV(auxStack.size());
}

void FtParser::completeReduction() {
  LOGSECTION("FtParser::completeReduction()");
  int token = ibnfs[ibnfb[ruleToReduce]+reductionSelection];
  completeReduction(token);
}

void FtParser::parseAction() {
  LOGSECTION("FtParser::parseAction");
  state_number_map *sp = &map_state_number[state.number];
  unsigned *tokenPointer = lstptr(*sp, t_actions);

  if (processState < running) {
    processState = running;
  }
  int k = 0;
  while (tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
    k++;
  }

  actionParameter = lstptr(*sp, p_actions)[k];
  ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();

  if (processState <= running && state.token == 0) {
    getToken();
  }
  stackChanged(stateStack.size()+1);
}

void FtParser::stepToken(unsigned token) {
  LOGSECTION("FtParser::stepToken");
  LOGV(state.number);
  LOGV(state.token) LCV((int) state.pointer);
  lookAhead = state.pointer = 0;
  inputToken = token;
  processState = running;

  if (map_token_number[token].non_terminal_flag) {
    reductionState = state;
    reductionState.token = token;
    state.token = inputToken = 0;
    dispatchReductionToken();
    transStack.discardData();
    auxStack.discardData();
    if (processState <= running && state.token == 0) {
      getToken();
    }
    stackChanged(stateStack.size());
    return;
  }
  inputToken = token;
  state_number_map *sp = &map_state_number[state.number];
  unsigned *tokenPointer = lstptr(*sp, t_actions);
  state.token = token;

  int k = 0;
  while(tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
    k++;
  }

  actionParameter = lstptr(*sp, p_actions)[k];
  ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
  LOGV(processState) LCV(state.token);
  if (processState <= running && state.token == 0) {
    getToken();
  }
  stackChanged(stateStack.size()+1);
}

void FtParser::parseToken(unsigned token) {
  LOGSECTION("FtParser::parseToken");
  LOGV(state.number) LCV(token);
  LOGV(state.token) LCV((int) state.pointer);
  lookAhead = state.pointer = 0;
  inputToken = token;
  processState = token ? running : syntaxError;
  if (processState == syntaxError) {
    return;
  }

  if (map_token_number[token].non_terminal_flag) {
    reductionState = state;
    reductionState.token = token;
    state.token = inputToken = 0;
    dispatchReductionToken();
    if (processState <= running && state.token == 0) {
      getToken();
    }
    return;
  }
  inputToken = state.token = token;

  while (processState <= running && inputToken != 0) {
    state_number_map *sp = &map_state_number[state.number];
    unsigned *tokenPointer = lstptr(*sp, t_actions);
    int k = 0;
    while(tokenPointer[k] != (unsigned) state.token && tokenPointer[k]) {
      k++;
    }
    LOGV(k) LCV(sp->n_actions);
    assert((unsigned) k < sp->n_actions);
    actionParameter = lstptr(*sp, p_actions)[k];
    ((*this).*( terminalAction[lstptr(*sp, a_actions)[k]] ))();
    LOGV(processState) LCV(state.token);
    if (processState <= running && inputToken != 0) {
      state.token = inputToken;
    }
  }
  if (processState <= running && state.token == 0) {
    getToken();
  }
  stackChanged(stateStack.size()+1);
}



int FtParser::validToken(unsigned token, unsigned sn) {
  LOGSECTION("FtParser::validToken");
  state_number_map *sp = &map_state_number[sn];
  unsigned *tokenPointer = lstptr(*sp, t_actions);

  int k = 0;
  while (tokenPointer[k] && tokenPointer[k] != token && tokenPointer[k]) {
    k++;
  }
  if (k == 0 && lstptr(*sp, a_actions)[k] == pe_syn_error) {
    return 0;
  }
  LOGV(sn);
  LOGV(token);
  LOGV(k);
  LOGV(tokenPointer[k]);
  return 1;
}

int FtParser::validSelection(unsigned selection, unsigned sn) {
  LOGSECTION("FtParser::validSelection");
  state_number_map *sp = &map_state_number[sn];
  unsigned *tokenPointer = lstptr(*sp, t_actions);

  unsigned token = ibnfs[ibnfb[ruleToReduce] + selection];

  int k;
  for (k = sp->n_actions; k && tokenPointer[--k] != token; ) {
    LOGV(k);
    LOGV(tokenPointer[k]);
  }
  LOGV(sn);
  LOGV(token);
  LOGV(k);
  LOGV(tokenPointer[k]);
  return k != 0;
}

FtParser &FtParser::parseTo(unsigned char *target) {
  LOGSECTION("FtParser::parseTo");
  int backup = 0;
  while (stateStack.size() && state.pointer <= target) {
    stateStack.pop(state);
    backup = 1;
  }
  if (backup) {
    processState = running;
    lookAhead = state.pointer;
    auxStack.discardData();
    transStack.discardData();
    LOGV(auxStack.size());
    nNullShifts = 0;
    getToken();
  }
  else if (processState == finished) {
    reset();
  }
  if (processState < running) {
    processState = running;
  }
  LOGV(state.token) LCV((int) state.pointer);
  while (processState <= running && state.pointer < target) {
    parseAction();
  }
  //if ((dc *)displayControl) {
  //  displayControl->des->d_size.y = stateStack.size()+1;
  //}
  stackChanged(stateStack.size()+1);
  return *this;
}

FtParser &FtParser::parse() {
  if (processState < running) {
    processState = running;
  }
  while (processState <= running) {
    parseAction();
  }
  //if ((dc *)displayControl) {
  //  displayControl->des->d_size.y = stateStack.size()+1;
  //}
  stackChanged(stateStack.size()+1);
  return *this;
}

FtParser &FtParser::parse(const char *fragment) {
  LOGSECTION("FtParser::parse(const char *)");
  LOGV(fragment);
  lookAhead = state.pointer = (const unsigned char *) fragment;
  inputToken = 0;
  endPointer = (const unsigned char *) fragment + strlen(fragment);
  getToken();
  LOGV(processState) LCV(inputToken);

  while (processState <= running) {
    parseAction();
    LOGV(state.pointer);
  }
  //if ((dc *)displayControl) {
  //  displayControl->des->d_size.y = stateStack.size()+1;
  //}
  stackChanged(stateStack.size()+1);
  if (processState == unexpectedEndOfFile) {
    processState = ready;
  }
  return *this;
}

FtParser &FtParser::prime(const char *fragment) {
  LOGSECTION("FtParser::parse(const char *)");
  LOGV(fragment);
  lookAhead = state.pointer = (const unsigned char *) fragment;
  endPointer = (const unsigned char *) fragment + strlen(fragment);
  getToken();
  LOGV(processState);
  stackChanged(stateStack.size()+1);
  return *this;
}

FtParser &FtParser::parseTo(cint *loc) {
  LOGSECTION("FtParser::parseTo");
  LOGV(*loc);
  LOGV(stateStack.size());
  LOGV(location());
  int backup = 0;
  LOGV(processState);
  while (stateStack.size()
	 && (loc->y < state.line
	     || (loc->y == state.line && loc->x <= state.charPos))) {
    stateStack.pop(state);
    backup = 1;
  }
  LOGV(*loc) LCV(location());
  LOGV(precedes(*loc, location()));
  if (backup) {
    processState = running;
    lookAhead = state.pointer;
    getToken();
  }
  else if (precedes(*loc, location())) {
    reset();
  }
  LOGV(backup);
  LOGV(processState);
  LOGV(stateStack.size());
  LOGV(state.number) LCV((int) state.pointer);
  LOGV(location());
  if (processState < running) {
    processState = running;
  }
  while (processState == running &&
         (state.line < loc->y || (state.line == loc->y &&
				  state.charPos < loc->x))) {
    parseAction();
  }
  LOGV(location());
  //if ((dc *)displayControl) {
  //  displayControl->des->d_size.y = stateStack.size()+1;
  //}
  stackChanged(stateStack.size()+1);
  return *this;
}

FtParser &FtParser::step() {
  LOGSECTION("FtParser::step");
  LOGV(location());
  if (processState < running) {
    processState = running;
  }
  if (processState == running) {
    parseAction();
  }
  //if ((dc *)displayControl) {
  //  displayControl->des->d_size.y = stateStack.size()+1;
  //}
  stackChanged(stateStack.size()+1);
  return *this;
}

FtParser &FtParser::step(char *fragment) {
  LOGSECTION("FtParser::step(const char *)");
  LOGV(fragment);
  lookAhead = state.pointer = (unsigned char *) fragment;
  endPointer = (unsigned char *) fragment + strlen(fragment);
  getToken();
  LOGV(processState);
  if (processState <= running) {
    parseAction();
  }
  //if ((dc *)displayControl) {
  //  displayControl->des->d_size.y = stateStack.size()+1;
  //}
  stackChanged(stateStack.size()+1);
  if (processState <= running && state.token == 0) {
    getToken();
  }
  if (processState == unexpectedEndOfFile) {
    processState = ready;
  }
  return *this;
}

const char *FtParser::processStateText[] = {
  "Ready",
  "Ready",                                   //running,
  "Parse complete",                          //finished,
  "Syntax error",           //syntaxError,
  "Unexpected end of file", //unexpectedEndOfFile,
  "Select reduction token",  //selectionRequired
  "Selection error"
};

#if 0 /* unused */
tsd *x1x_new(pcb_type *pcb) {
  int sx, sn, tn, /*fn,*/ fx;
  int *items;
  int nitems;
  tsd *isl = init_tsd(4);

  ok_ptr(pcb);
  sn = PCB.s.sn;
  tn = PCB.token_number;
  if (PCB.exit_flag) {
    tn = 0;
  }
  sx = PCB.ssx;
  if (PCB.reduction_token) {
    sx -= PCB.rule_length;
  }

  {
    tuple_dict *d;
    d = xis(sn);
    items = d->text;
    nitems = d->nsx;
    items += 2*nitems;
    while (nitems--) {
      fx = *--items;
      Rule rule = *--items;
      if (tn == 0 ||
          fx >= rule->non_vanishing_length ||
          rule.token(fx).isExpansionToken(tn)) {
	//x2(Rule(fn)->token(fx), tn))
	//x2(lstptr(map_form_number[fn],tokens)[fx], tn))
        at(isl,sx,sn,(int) rule,fx);
      }
    }
    delete_tuple_dict(d);
  }

  while (sx-- > 0) {
    tuple_dict *d;
    tn = map_state_number[sn].char_token;
    sn = PCB.ss[sx].sn;
    d = xis(sn);
    items = d->text;
    nitems = d->nsx;
    items += 2*nitems;
    while (nitems--) {
      fx = *--items;
      Rule rule = *--items;
      if (fx >= rule->length()) {
	continue;
      }
      if (x3(isl, sx, (int)rule, fx)) {
	at(isl, sx, sn, (int) rule, fx);
      }
    }
    delete_tuple_dict(d);
  }
  return isl;
}
#endif /* 0 - unused */

tsd *FtParser::x1x_new() {
  LOGSECTION("FtParser::x1x_new");
  int sx, sn, fn, fx;
  tsd *isl = init_tsd(4);

  sn = state.number;
  Token tn = state.token;
  if (processState == selectionRequired) {
    sn = reductionState.number;
    tn = reductionState.token;
  }
  else if (processState == syntaxError) {
    tn = 0;
  }
  tuple_dict *d = xis(sn);
  int *items = d->text;
  int nitems = d->nsx;
  items += 2*nitems;
  LOGV(state.number);
  LOGV(state.token);
  LOGV(stateStack.size());
  LOGV(processState);
  LOGV(ntkns);
  sx = stateStack.size();
  if (processState == selectionRequired) {
    int rx = Rule(ruleToReduce)->length();
    int n = stateStack.size();
    for (sx = reductionIndex; rx >= 0; rx--) {
      int index = sx + rx;
      if (index > n) {
	continue;
      }
      if (index < n) {
	sn = stateStack[index].number;
      }
      at(isl, index, sn, ruleToReduce, rx);
    }
    tuple_dict *d = xis(reductionState.number);
    int *items = d->text;
    int nitems = d->nsx;
    items += 2*nitems;
    while (nitems--) {
      fx = *--items;
      fn = *--items;
      LOGV(fn);
      LOGV(fx);
      if ((unsigned) fx >= Rule(fn)->length()) {
	continue;
      }
      if (x3a(isl, sx, fn, fx)) {
	at(isl, sx, sn, fn, fx);
      }
    }
  }
  else {
    state_number_map *sp = &map_state_number[sn];
    unsigned *tokenPointer = lstptr(*sp, t_actions);

    unsigned k = 0;
    while (tokenPointer[k] && 
	   tokenPointer[k] != (unsigned) tn && tokenPointer[k]) {
      k++;
    }
    if (k == 0 && lstptr(*sp, a_actions)[k] == pe_syn_error) {
      tn = 0;
    }
    LOGV(k);
    LOGV(tn);
    LOGV(nitems);
    if (tn.isNotNull()) {
      while (nitems--) {
        fx = *--items;
        fn = *--items;
        LOGV(fn);
        LOGV(fx);
        //if (x4(tn, fn)) {
        if (tn.isExpansionRule(fn)) {
          at(isl, sx, sn, fn, fx);
          continue;
        }
        Rule rule = fn;
        if ((unsigned) fx >= rule->length()) {
	  continue;
	}
        //if (lstptr(map_form_number[fn],tokens)[fx] == tn) {
	//  at(isl, sx, sn, fn, fx);
	//}
        if (rule.token(fx) == tn) {
	  at(isl, sx, sn, (int) rule, fx);
	}
      }
      LOGV(isl->nt);
      for (k = 0; k < sp->n_completed_forms; k++) {
        unsigned rule = lstptr(*sp,completed_forms)[k];
        LOGV(rule);
        at(isl, sx, sn, rule, Rule(rule)->length());
      }
      LOGV(isl->nt);
      if (isl->nt) {
        items = d->text;
        nitems = d->nsx;
        items += 2*nitems;
        while (nitems--) {
          fx = *--items;
          fn = *--items;
          LOGV(fn);
          LOGV(fx);
          if ((unsigned) fx >= Rule(fn)->length()) {
	    continue;
	  }
          if (x3a(isl, sx, fn, fx)) {
	    at(isl, sx, sn, fn, fx);
	  }
        }
      }
      else {
        items = d->text;
        nitems = d->nsx;
        items += 2*nitems;
        while (nitems--) {
          fx = *--items;
          fn = *--items;
          LOGV(fn);
          LOGV(fx);
          at(isl, sx, sn, fn, fx);
        }
      }
    }
    else while (nitems--) {
      fx = *--items;
      fn = *--items;
      LOGV(fn);
      LOGV(fx);
      at(isl, sx, sn, fn, fx);
    }
  }
  LOGV(isl->nt);
  LOGV((int) d);
  delete_tuple_dict(d);
  LOGV(sx);
  while (sx-- > 0) {
    tuple_dict *d;
    tn = stateStack[sx].token;
    sn = stateStack[sx].number;
    LOGV(sn);
    LOGV(tn);
    d = xis(sn);
    items = d->text;
    nitems = d->nsx;
    items += 2*nitems;
    while (nitems--) {
      fx = *--items;
      fn = *--items;
      LOGV(fn);
      LOGV(fx);

      if ((unsigned) fx >= Rule(fn)->length()) {
	continue;
      }
      if (x3(isl, sx, fn, fx)) {
	at(isl, sx, sn, fn, fx);
      }
    }
    delete_tuple_dict(d);
  }
  LOGV(isl->nt);
  return isl;
}