--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/examples/mpp/token.h	Sat Dec 22 17:52:45 2007 -0500
@@ -0,0 +1,258 @@
+/*
+ * AnaGram, a System for Syntax Directed Programming
+ * C Macro preprocessor
+ * Token handling definitions
+ *
+ * Copyright 1993 Parsifal Software. All Rights Reserved.
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "charsink.h"
+#include "strdict.h"
+
+extern string_dictionary td;
+
+
+// define token identification codes
+
+enum token_id {
+  END_OF_FILE       =0,
+  SPACE             =' ',
+  ANDAND            ='A', // "&&"
+  ANDassign,              // "&="
+  ARROW,                  // "->"
+  CONCAT,                 // "##"
+  DECR,                   // "--"
+  DIVassign,              // "/="
+  ELLIPSIS,               // "..."
+  EQ,                     // "=="
+  ERassign,               // "^="
+  GE,                     // ">="
+  ICR,                    // "++"
+  LE,                     // "<="
+  LS,                     // "<<"
+  LSassign,               // "<<="
+  MODassign,              // "%="
+  MINUSassign,            // "-="
+  MULTassign,             // "*="
+  NE,                     // "!="
+  ORassign,               // "|="
+  OROR,                   // "||"
+  PLUSassign,             // "+="
+  RS,                     // ">>"
+  RSassign,               // ">>="
+  CHARACTERconstant,      // character constant
+  STRINGliteral,          // character string
+  HEXconstant       =129,
+  OCTconstant,
+  DECconstant,
+  FLOATconstant,          // real
+  NAME,
+  AUTO,                   // "auto"
+  BREAK,                  // "break"
+  CASE,                   // "case"
+  CHAR,                   // "char"
+  CONSTANT,               // "const"
+  CONTINUE,               // "continue"
+  DEFAULT,                // "default"
+  DO,                     // "do"
+  DOUBLE,                 // "double"
+  ELSE,                   // "else"
+  ENUM,                   // "enum"
+  EXTERN,                 // "extern"
+  FLOAT,                  // "float"
+  FOR,                    // "for"
+  GOTO,                   // "goto"
+  IF,                     // "if"
+  INT,                    // "int"
+  LONG,                   // "long"
+  REGISTER,               // "register"
+  RETURN,                 // "return"
+  SHORT,                  // "short"
+  SIGNED,                 // "signed"
+  SIZEOF,                 // "sizeof"
+  STATIC,                 // "static"
+  STRUCT,                 // "struct"
+  SWITCH,                 // "switch"
+  TYPEDEF,                // "typedef"
+  UNION,                  // "union"
+  UNSIGNED,               // "unsigned"
+  VOIDkey,                // "void"
+  VOLATILE,               // "volatile"
+  WHILE,                  // "while"
+  UNRECOGNIZED
+};
+
+struct token {
+  unsigned handle;                   // token dictionary index
+  token_id id;                       // token type
+};
+
+// Token sink class
+
+class token_sink {
+public:
+  virtual ~token_sink() {}
+  virtual token_sink &operator << (token) = 0;
+  virtual token_sink &operator << (token *) = 0;
+  friend int error(token_sink &);
+};
+
+
+// Token accumulator
+//   Member functions that are not inline are in token.cpp
+
+class token_accumulator : public token_sink {
+  token *cs;
+  unsigned *xs;
+  unsigned csx;
+  unsigned xsx;
+  unsigned csmax;
+  unsigned xsmax;
+public:
+
+// Constructor
+
+  token_accumulator(int nc, int nx = 1);
+
+
+// Destructor
+
+  ~token_accumulator();
+
+
+// Reset token accumulator
+
+  friend token_accumulator &reset(token_accumulator &);
+
+
+// Change levels
+
+  token_accumulator& operator ++(){
+    assert(xsx);                        // Room for another level?
+    xs[--xsx] = csx;
+    return *this;
+  }
+
+  token_accumulator& operator --() {
+    csx = xs[xsx++];
+    assert(xsx < xsmax);                // Have to leave last level active
+    return *this;
+  }
+
+
+// Push data
+
+  token_sink& operator << (token c) {
+    assert(csx < csmax);                // room for data?
+    cs[csx++] = c;
+    return *this;
+  }
+
+  token_sink &operator << (token *tp);
+
+  token_sink &operator << (token_accumulator &t) {
+    assert(this != &t);                 // prevent disaster
+    return *this << (token *) t;
+  }
+
+// Pop data
+
+  token_accumulator &operator >> (token &c);
+
+
+// Access data
+
+  //token &operator [](unsigned);
+
+  operator token *() {
+    cs[csx].id = END_OF_FILE;
+    cs[csx].handle = 0;
+    return &cs[xs[xsx]];
+  }
+
+
+// Concatenate token strings
+
+  friend token_accumulator& concat(token_accumulator &);
+
+
+// Make permanent copy of token string
+
+  friend token *copy(token_accumulator &);
+
+
+// Get size of token string
+
+  friend unsigned size(token_accumulator &s);
+
+// Check for error
+
+  friend int error(token_accumulator &t);
+
+};
+
+  inline unsigned size(token_accumulator &s) {
+    return s.csx - s.xs[s.xsx];
+  }
+
+  inline int error(token_accumulator &t) {
+    return t.csx >= t.csmax || t.xsx >= t.xsmax;
+  }
+
+
+
+// Token Translator class
+
+class token_translator : public token_sink {
+  character_sink *cs;
+  int space_required;
+public:
+
+
+// Constructor
+
+  token_translator(character_sink *sink) {
+    cs = sink;
+    space_required = 0;
+  }
+
+
+// Accept data
+
+  token_sink &operator << (token t) {
+    if (space_required && t.id > 128) *cs << ' ';
+    *cs << td[t.handle];
+    space_required = t.id > 128;
+    return *this;
+  }
+
+  token_sink &operator << (token *tp);
+
+
+// Check for error;
+  friend int error(token_sink &) {return 0;}
+};
+
+#endif