diff src/share/classes/com/sun/xml/internal/rngom/parse/compact/CompactSyntax.jj @ 0:0961a4a21176

Initial load
author duke
date Sat, 01 Dec 2007 00:00:00 +0000
parents
children
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/classes/com/sun/xml/internal/rngom/parse/compact/CompactSyntax.jj	Sat Dec 01 00:00:00 2007 +0000
@@ -0,0 +1,1963 @@
+options {
+  STATIC = false;
+  UNICODE_INPUT = true;
+  JAVA_UNICODE_ESCAPE = true;
+}
+
+PARSER_BEGIN(CompactSyntax)
+
+package com.sun.xml.internal.rngom.parse.compact;
+
+import java.io.Reader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.Hashtable;
+import java.util.List;
+
+import com.sun.xml.internal.rngom.ast.builder.Annotations;
+import com.sun.xml.internal.rngom.ast.builder.BuildException;
+import com.sun.xml.internal.rngom.ast.builder.CommentList;
+import com.sun.xml.internal.rngom.ast.builder.DataPatternBuilder;
+import com.sun.xml.internal.rngom.ast.builder.Div;
+import com.sun.xml.internal.rngom.ast.builder.ElementAnnotationBuilder;
+import com.sun.xml.internal.rngom.ast.builder.Grammar;
+import com.sun.xml.internal.rngom.ast.builder.GrammarSection;
+import com.sun.xml.internal.rngom.ast.builder.Include;
+import com.sun.xml.internal.rngom.ast.builder.IncludedGrammar;
+import com.sun.xml.internal.rngom.ast.builder.NameClassBuilder;
+import com.sun.xml.internal.rngom.ast.builder.SchemaBuilder;
+import com.sun.xml.internal.rngom.ast.builder.Scope;
+import com.sun.xml.internal.rngom.ast.om.Location;
+import com.sun.xml.internal.rngom.ast.om.ParsedElementAnnotation;
+import com.sun.xml.internal.rngom.ast.om.ParsedNameClass;
+import com.sun.xml.internal.rngom.ast.om.ParsedPattern;
+import com.sun.xml.internal.rngom.parse.Context;
+import com.sun.xml.internal.rngom.parse.IllegalSchemaException;
+import com.sun.xml.internal.rngom.parse.Parseable;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import org.xml.sax.helpers.LocatorImpl;
+
+import com.sun.xml.internal.rngom.util.Localizer;
+import com.sun.xml.internal.rngom.xml.util.WellKnownNamespaces;
+
+
+public class CompactSyntax implements Context {
+  private static final int IN_ELEMENT = 0;
+  private static final int IN_ATTRIBUTE = 1;
+  private static final int IN_ANY_NAME = 2;
+  private static final int IN_NS_NAME = 4;
+
+  private String defaultNamespace;
+  private String compatibilityPrefix = null;
+  private SchemaBuilder sb;
+  private NameClassBuilder ncb;
+  private String sourceUri;
+  /**
+   * This is what we are parsing right now.
+   */
+  private CompactParseable parseable;
+  private ErrorHandler eh;
+  private final Hashtable namespaceTable = new Hashtable();
+  private final Hashtable datatypesTable = new Hashtable();
+  private boolean hadError = false;
+  private static final Localizer localizer = new Localizer(new Localizer(Parseable.class),CompactSyntax.class);
+  private final Hashtable attributeNameTable = new Hashtable();
+  private boolean annotationsIncludeElements = false;
+  
+  /**
+   * String that represents the inherited namespace.
+   * 
+   * <p>
+   * HACK: we always allocate a new String instance so that
+   * we can distinguish inherited value from the explicitly
+   * given value.
+   */
+  private /*final*/ String inheritedNs; // essentially final but JavaCC don't let us declare it as so.
+  
+  final class LocatedString {
+    private final String str;
+    private final Token tok;
+
+    LocatedString(String str, Token tok) {
+      this.str = str;
+      this.tok = tok;
+    }
+
+    String getString() {
+      return str;
+    }
+
+    Location getLocation() {
+      return makeLocation(tok);
+    }
+
+    Token getToken() {
+      return tok;
+    }
+
+  }
+
+  public CompactSyntax(CompactParseable parseable, Reader r, String sourceUri, SchemaBuilder sb, ErrorHandler eh, String inheritedNs) {
+    this(r);
+    this.sourceUri = sourceUri;
+    this.parseable = parseable;
+    this.sb = sb;
+    this.ncb = sb.getNameClassBuilder();
+    this.eh = eh;
+    // this causes the root pattern to have non-null annotations
+    // which is useful because it gives a context to trang
+    this.topLevelComments = sb.makeCommentList();
+    this.inheritedNs = defaultNamespace = new String(inheritedNs);
+  }
+
+  ParsedPattern parse(Scope scope) throws IllegalSchemaException {
+    try {
+      ParsedPattern p = Input(scope);
+      if (!hadError)
+        return p;
+    }
+    catch (ParseException e) {
+      error("syntax_error", e.getMessage(), e.currentToken.next);
+    }
+    catch (EscapeSyntaxException e) {
+      reportEscapeSyntaxException(e);
+    }
+    throw new IllegalSchemaException();
+  }
+
+  ParsedPattern parseInclude(IncludedGrammar g) throws IllegalSchemaException {
+    try {
+      ParsedPattern p = IncludedGrammar(g);
+      if (!hadError)
+        return p;
+    }
+    catch (ParseException e) {
+      error("syntax_error", e.getMessage(), e.currentToken.next);
+    }
+    catch (EscapeSyntaxException e) {
+      reportEscapeSyntaxException(e);
+    }
+    throw new IllegalSchemaException();
+  }
+
+  private void checkNsName(int context, LocatedString ns) {
+    if ((context & IN_NS_NAME) != 0)
+      error("ns_name_except_contains_ns_name", ns.getToken());
+  }
+
+  private void checkAnyName(int context, Token t) {
+    if ((context & IN_NS_NAME) != 0)
+      error("ns_name_except_contains_any_name", t);
+    if ((context & IN_ANY_NAME) != 0)
+      error("any_name_except_contains_any_name", t);
+  }
+
+  private void error(String key, Token tok) {
+    doError(localizer.message(key), tok);
+  }
+
+  private void error(String key, String arg, Token tok) {
+    doError(localizer.message(key, arg), tok);
+  }
+
+  private void error(String key, String arg1, String arg2, Token tok) {
+    doError(localizer.message(key, arg1, arg2), tok);
+  }
+
+  private void doError(String message, Token tok) {
+    hadError = true;
+    if (eh != null) {
+      LocatorImpl loc = new LocatorImpl();
+      loc.setLineNumber(tok.beginLine);
+      loc.setColumnNumber(tok.beginColumn);
+      loc.setSystemId(sourceUri);
+      try {
+	eh.error(new SAXParseException(message, loc));
+      }
+      catch (SAXException se) {
+	throw new BuildException(se);
+      }
+    }
+  }
+
+  private void reportEscapeSyntaxException(EscapeSyntaxException e) {
+    if (eh != null) {
+      LocatorImpl loc = new LocatorImpl();
+      loc.setLineNumber(e.getLineNumber());
+      loc.setColumnNumber(e.getColumnNumber());
+      loc.setSystemId(sourceUri);
+      try {
+	eh.error(new SAXParseException(localizer.message(e.getKey()), loc));
+      }
+      catch (SAXException se) {
+	throw new BuildException(se);
+      }
+    }
+  }
+
+  private static String unquote(String s) {
+    if (s.length() >= 6 && s.charAt(0) == s.charAt(1)) {
+      s = s.replace('\u0000', '\n');
+      return s.substring(3, s.length() - 3);
+    }
+    else
+      return s.substring(1, s.length() - 1);
+  }
+
+  Location makeLocation(Token t) {
+    return sb.makeLocation(sourceUri, t.beginLine, t.beginColumn);
+  }
+
+  private static ParsedPattern[] addPattern(ParsedPattern[] patterns, int i, ParsedPattern p) {
+    if (i >= patterns.length) {
+      ParsedPattern[] oldPatterns = patterns;
+      patterns = new ParsedPattern[oldPatterns.length*2];
+      System.arraycopy(oldPatterns, 0, patterns, 0, oldPatterns.length);
+    }
+    patterns[i] = p;
+    return patterns;
+  }
+
+  String getCompatibilityPrefix() {
+    if (compatibilityPrefix == null) {
+      compatibilityPrefix = "a";
+      while (namespaceTable.get(compatibilityPrefix) != null)
+        compatibilityPrefix = compatibilityPrefix + "a";
+    }
+    return compatibilityPrefix;
+  }
+
+  public String resolveNamespacePrefix(String prefix) {
+    String result = (String)namespaceTable.get(prefix);
+    if (result.length() == 0)
+      return null;
+    return result;
+  }
+
+  public Enumeration prefixes() {
+    return namespaceTable.keys();
+  }
+
+  public String getBaseUri() {
+    return sourceUri;
+  }
+
+  public boolean isUnparsedEntity(String entityName) {
+    return false;
+  }
+
+  public boolean isNotation(String notationName) {
+    return false;
+  }
+
+  public Context copy() {
+    return this;
+  }
+
+  private Context getContext() {
+    return this;
+  }
+
+  private CommentList getComments() {
+    return getComments(getTopLevelComments());
+  }
+
+  private CommentList topLevelComments;
+
+  private CommentList getTopLevelComments() {
+    CommentList tem = topLevelComments;
+    topLevelComments = null;
+    return tem;
+  }
+
+  private void noteTopLevelComments() {
+    topLevelComments = getComments(topLevelComments);
+  }
+
+  private void topLevelComments(GrammarSection section) {
+    section.topLevelComment(getComments(null));
+  }
+
+  private Token lastCommentSourceToken = null;
+
+  private CommentList getComments(CommentList comments) {
+    Token nextToken = getToken(1);
+    if (lastCommentSourceToken != nextToken) {
+      if (lastCommentSourceToken == null)
+        lastCommentSourceToken = token;
+      do {
+	lastCommentSourceToken = lastCommentSourceToken.next;
+	Token t = lastCommentSourceToken.specialToken;
+	if (t != null) {
+	  while (t.specialToken != null)
+	    t = t.specialToken;
+	  if (comments == null)
+	    comments = sb.makeCommentList();
+	  for (; t != null; t = t.next) {
+            String s = mungeComment(t.image);
+            Location loc = makeLocation(t);
+            if (t.next != null
+                && t.next.kind == CompactSyntaxConstants.SINGLE_LINE_COMMENT_CONTINUE) {
+              StringBuffer buf = new StringBuffer(s);
+              do {
+                t = t.next;
+                buf.append('\n');
+                buf.append(mungeComment(t.image));
+              } while (t.next != null
+                       && t.next.kind == CompactSyntaxConstants.SINGLE_LINE_COMMENT_CONTINUE);
+              s = buf.toString();
+            }
+	    comments.addComment(s, loc);
+          }
+	}
+      } while (lastCommentSourceToken != nextToken);
+    }
+    return comments;
+  }
+
+  private ParsedPattern afterComments(ParsedPattern p) {
+    CommentList comments = getComments(null);
+    if (comments == null)
+      return p;
+    return sb.commentAfter(p, comments);
+  }
+
+  private ParsedNameClass afterComments(ParsedNameClass nc) {
+    CommentList comments = getComments(null);
+    if (comments == null)
+      return nc;
+    return ncb.commentAfter(nc, comments);
+  }
+
+  private static String mungeComment(String image) {
+    int i = image.indexOf('#') + 1;
+    while (i < image.length() && image.charAt(i) == '#')
+      i++;
+    if (i < image.length() && image.charAt(i) == ' ')
+      i++;
+    return image.substring(i);
+  }
+
+  private Annotations getCommentsAsAnnotations() {
+    CommentList comments = getComments();
+    if (comments == null)
+      return null;
+    return sb.makeAnnotations(comments, getContext());
+  }
+
+  private Annotations addCommentsToChildAnnotations(Annotations a) {
+    CommentList comments = getComments();
+    if (comments == null)
+      return a;
+    if (a == null)
+      a = sb.makeAnnotations(null, getContext());
+    a.addComment(comments);
+    return a;
+  }
+
+  private Annotations addCommentsToLeadingAnnotations(Annotations a) {
+    CommentList comments = getComments();
+    if (comments == null)
+      return a;
+    if (a == null)
+      return sb.makeAnnotations(comments, getContext());
+    a.addLeadingComment(comments);
+    return a;
+  }
+
+  private Annotations getTopLevelCommentsAsAnnotations() {
+    CommentList comments = getTopLevelComments();
+    if (comments == null)
+      return null;
+    return sb.makeAnnotations(comments, getContext());
+  }
+
+  private void clearAttributeList() {
+    attributeNameTable.clear();
+  }
+
+  private void addAttribute(Annotations a, String ns, String localName, String prefix, String value, Token tok) {
+    String key = ns + "#" + localName;
+    if (attributeNameTable.get(key) != null)
+      error("duplicate_attribute", ns, localName, tok);
+    else {
+      attributeNameTable.put(key, key);
+      a.addAttribute(ns, localName, prefix, value, makeLocation(tok));
+    }
+  }
+
+  private void checkExcept(Token[] except) {
+    if (except[0] != null)
+      error("except_missing_parentheses", except[0]);
+  }
+
+  private String lookupPrefix(String prefix, Token t) {
+    String ns = (String)namespaceTable.get(prefix);
+    if (ns == null) {
+      error("undeclared_prefix", prefix, t);
+      return "#error";
+    }
+    return ns;
+  }
+  private String lookupDatatype(String prefix, Token t) {
+    String ns = (String)datatypesTable.get(prefix);
+    if (ns == null) {
+      error("undeclared_prefix", prefix, t);
+      return ""; // XXX
+    }
+    return ns;
+  }
+  private String resolve(String str) {
+    try {
+     return new URL(new URL(sourceUri), str).toString();
+    }
+    catch (MalformedURLException e) { }
+    return str;
+  }
+}
+
+PARSER_END(CompactSyntax)
+
+ParsedPattern Input(Scope scope) :
+{
+  ParsedPattern p;
+}
+{
+  Preamble()
+  (LOOKAHEAD(TopLevelLookahead()) p = TopLevelGrammar(scope)
+   | p = Expr(true, scope, null, null) { p = afterComments(p); } <EOF>)
+  { return p; }
+}
+
+void TopLevelLookahead() :
+{}
+{
+  <PREFIXED_NAME> "["
+  | Identifier() ("[" | "=" | "&=" | "|=")
+  | LookaheadGrammarKeyword()
+  | LookaheadBody() LookaheadAfterAnnotations()
+  | LookaheadDocumentation() (LookaheadBody())? LookaheadAfterAnnotations()
+}
+
+void LookaheadAfterAnnotations() :
+{}
+{
+  Identifier() ("=" | "&=" | "|=")
+  | LookaheadGrammarKeyword()
+}
+
+void LookaheadGrammarKeyword() :
+{}
+{
+  "start" | "div" | "include"
+}
+
+void LookaheadDocumentation() :
+{}
+{
+  ((<DOCUMENTATION> | <DOCUMENTATION_AFTER_SINGLE_LINE_COMMENT>) (<DOCUMENTATION_CONTINUE>)*)+
+}
+
+void LookaheadBody() :
+{}
+{
+  "["
+   (<PREFIXED_NAME> | UnprefixedName() | "=" | <LITERAL> | "~" | LookaheadBody() )*
+  "]"
+}
+
+ParsedPattern IncludedGrammar(IncludedGrammar g) :
+{
+  Annotations a;
+  ParsedPattern p;
+}
+{
+  Preamble()
+  (LOOKAHEAD(TopLevelLookahead()) a = GrammarBody(g, g, getTopLevelCommentsAsAnnotations())
+   | a = Annotations() "grammar" "{" a = GrammarBody(g, g, a) { topLevelComments(g); } "}")
+  { p = afterComments(g.endIncludedGrammar(sb.makeLocation(sourceUri, 1, 1), a)); }
+  <EOF>
+  { return p; }
+}
+
+ParsedPattern TopLevelGrammar(Scope scope) :
+{
+  Annotations a = getTopLevelCommentsAsAnnotations();
+  Grammar g;
+  ParsedPattern p;
+}
+{
+  { g = sb.makeGrammar(scope); }
+  a = GrammarBody(g, g, a)
+  { p = afterComments(g.endGrammar(sb.makeLocation(sourceUri, 1, 1), a)); }
+  <EOF>
+  { return p; }
+}
+
+void Preamble() :
+{}
+{
+  (NamespaceDecl() | DatatypesDecl())*
+  {
+    namespaceTable.put("xml", WellKnownNamespaces.XML);
+    if (datatypesTable.get("xsd") == null)
+      datatypesTable.put("xsd", WellKnownNamespaces.XML_SCHEMA_DATATYPES);
+  }
+}
+
+void NamespaceDecl() :
+{
+  LocatedString prefix = null;
+  boolean isDefault = false;
+  String namespaceName;
+}
+{
+  { noteTopLevelComments(); }
+  (("namespace" prefix = UnprefixedName())
+    | ("default" { isDefault = true; }
+      "namespace" (prefix = UnprefixedName())?))
+  "="
+  namespaceName = NamespaceName()
+  {
+    if (isDefault)
+      defaultNamespace = namespaceName;
+    if (prefix != null) {
+      if (prefix.getString().equals("xmlns"))
+        error("xmlns_prefix", prefix.getToken());
+      else if (prefix.getString().equals("xml")) {
+        if (!namespaceName.equals(WellKnownNamespaces.XML))
+          error("xml_prefix_bad_uri", prefix.getToken());
+      }
+      else if (namespaceName.equals(WellKnownNamespaces.XML))
+        error("xml_uri_bad_prefix", prefix.getToken());
+      else {
+        if (namespaceName.equals(WellKnownNamespaces.RELAX_NG_COMPATIBILITY_ANNOTATIONS))
+          compatibilityPrefix = prefix.getString();
+        namespaceTable.put(prefix.getString(), namespaceName);
+      }
+    }
+  }
+}
+
+String NamespaceName() :
+{
+  String r;
+}
+{
+  (r = Literal() | "inherit" { r = this.inheritedNs; })
+  { return r; }
+}
+
+void DatatypesDecl() :
+{
+  LocatedString prefix;
+  String uri;
+}
+{
+  { noteTopLevelComments(); }
+  "datatypes" prefix = UnprefixedName() "=" uri = Literal()
+  {
+    datatypesTable.put(prefix.getString(), uri);
+  }
+}
+
+ParsedPattern AnnotatedPrimaryExpr(boolean topLevel, Scope scope, Token[] except) :
+{
+  Annotations a;
+  ParsedPattern p;
+  ParsedElementAnnotation e;
+  Token t;
+}
+{
+  a = Annotations()
+  p = PrimaryExpr(topLevel, scope, a, except)
+  ( t = <FANNOTATE> e = AnnotationElement(false) {
+       if (topLevel)
+         error("top_level_follow_annotation", t);
+       else
+         p = sb.annotateAfter(p, e);
+     })*
+  { return p; }
+}
+
+
+ParsedPattern PrimaryExpr(boolean topLevel, Scope scope, Annotations a, Token[] except) :
+{
+  ParsedPattern p;
+}
+{
+  (p = ElementExpr(scope, a)
+   | p = AttributeExpr(scope, a)
+   | p = GrammarExpr(scope, a)
+   | p = ExternalRefExpr(scope, a)
+   | p = ListExpr(scope, a)
+   | p = MixedExpr(scope, a)
+   | p = ParenExpr(topLevel, scope, a)
+   | p = IdentifierExpr(scope, a)
+   | p = ParentExpr(scope, a)
+   | p = DataExpr(topLevel, scope, a, except)
+   | p = ValueExpr(topLevel, a)
+   | p = TextExpr(a)
+   | p = EmptyExpr(a)
+   | p = NotAllowedExpr(a))
+  { return p; }
+}
+
+ParsedPattern EmptyExpr(Annotations a) :
+{
+  Token t;
+}
+{
+  t = "empty"
+  { return sb.makeEmpty(makeLocation(t), a); }
+}
+
+ParsedPattern TextExpr(Annotations a) :
+{
+  Token t;
+}
+{
+  t = "text"
+  { return sb.makeText(makeLocation(t), a); }
+}
+
+ParsedPattern NotAllowedExpr(Annotations a) :
+{
+  Token t;
+}
+{
+  t = "notAllowed"
+  { return sb.makeNotAllowed(makeLocation(t), a); }
+}
+
+ParsedPattern Expr(boolean topLevel, Scope scope, Token t, Annotations a) :
+{
+  List patterns = new ArrayList();
+  ParsedPattern p;
+  boolean[] hadOccur = new boolean[1];
+  Token[] except = new Token[1];
+}
+{
+  p = UnaryExpr(topLevel, scope, hadOccur, except)
+  { patterns.add(p); }
+  (
+   { checkExcept(except); }
+   (t = "|" p = UnaryExpr(topLevel, scope, null, except)
+     { patterns.add(p); checkExcept(except); } )+
+    { p = sb.makeChoice(patterns, makeLocation(t), a); }
+    | (t = "&" p = UnaryExpr(topLevel, scope, null, except)
+     { patterns.add(p); checkExcept(except); } )+
+    { p = sb.makeInterleave(patterns, makeLocation(t), a); }
+    | (t = "," p = UnaryExpr(topLevel, scope, null, except)
+     { patterns.add(p); checkExcept(except); } )+
+    { p = sb.makeGroup(patterns, makeLocation(t), a); }
+  )?
+  {
+    if (patterns.size() == 1 && a != null) {
+      if (hadOccur[0])
+        p = sb.annotate(p, a);
+      else
+        p = sb.makeGroup(patterns, makeLocation(t), a);
+    }
+    return p;
+  }
+}
+
+ParsedPattern UnaryExpr(boolean topLevel, Scope scope, boolean[] hadOccur, Token[] except) :
+{
+  ParsedPattern p;
+  Token t;
+  ParsedElementAnnotation e;
+}
+{
+  p = AnnotatedPrimaryExpr(topLevel, scope, except)
+  (
+   {
+     if (hadOccur != null) hadOccur[0] = true;
+     p = afterComments(p);
+   }
+   (t = "+" { checkExcept(except); p = sb.makeOneOrMore(p, makeLocation(t), null); }
+    | t = "?" { checkExcept(except); p = sb.makeOptional(p, makeLocation(t), null); }
+    | t = "*" { checkExcept(except); p = sb.makeZeroOrMore(p, makeLocation(t), null); })
+   ( t = <FANNOTATE> e = AnnotationElement(false) {
+        if (topLevel)
+          error("top_level_follow_annotation", t);
+        else
+          p = sb.annotateAfter(p, e);
+     } )*
+  )?
+  { return p; }
+}
+
+ParsedPattern ElementExpr(Scope scope, Annotations a) :
+{
+  Token t;
+  ParsedNameClass nc;
+  ParsedPattern p;
+}
+{
+  t = "element"
+  nc = NameClass(IN_ELEMENT, null)
+  "{"
+  p = Expr(false, scope, null, null)
+  { p = afterComments(p); }
+  "}"
+  { return sb.makeElement(nc, p, makeLocation(t), a); }
+}
+
+ParsedPattern AttributeExpr(Scope scope, Annotations a) :
+{
+  Token t;
+  ParsedNameClass nc;
+  ParsedPattern p;
+}
+{
+  t = "attribute"
+  nc = NameClass(IN_ATTRIBUTE, null)
+  "{"
+  p = Expr(false, scope, null, null)
+  { p = afterComments(p); }
+  "}"
+  { return sb.makeAttribute(nc, p, makeLocation(t), a); }
+}
+
+ParsedNameClass NameClass(int context, Annotations[] pa) :
+{
+  Annotations a;
+  ParsedNameClass nc;
+}
+{
+  a = Annotations()
+  (nc = PrimaryNameClass(context, a) nc = AnnotateAfter(nc) nc = NameClassAlternatives(context, nc, pa)
+   | nc = AnyNameExceptClass(context, a, pa)
+   | nc = NsNameExceptClass(context, a, pa))
+  { return nc; }
+}
+
+ParsedNameClass AnnotateAfter(ParsedNameClass nc) :
+{
+  ParsedElementAnnotation e;
+}
+{
+  ( <FANNOTATE> e = AnnotationElement(false) { nc = ncb.annotateAfter(nc, e); })*
+  { return nc; }
+}
+
+ParsedNameClass NameClassAlternatives(int context, ParsedNameClass nc, Annotations[] pa) :
+{
+  Token t;
+  ParsedNameClass[] nameClasses;
+  int nNameClasses;
+}
+{
+  (
+    {
+      nameClasses = new ParsedNameClass[2];
+      nameClasses[0] = nc;
+      nNameClasses = 1;
+    }
+    (t = "|" nc = BasicNameClass(context) nc = AnnotateAfter(nc)
+    {
+      if (nNameClasses >= nameClasses.length) {
+	ParsedNameClass[] oldNameClasses = nameClasses;
+	nameClasses = new ParsedNameClass[oldNameClasses.length*2];
+	System.arraycopy(oldNameClasses, 0, nameClasses, 0, oldNameClasses.length);
+      }
+      nameClasses[nNameClasses++] = nc;
+    })+
+    {
+      Annotations a;
+      if (pa == null)
+        a = null;
+      else {
+        a = pa[0];
+        pa[0] = null;
+      }
+      nc = ncb.makeChoice(Arrays.asList(nameClasses).subList(0,nNameClasses), makeLocation(t), a);
+    }
+  )?
+  { return nc; }
+}
+
+ParsedNameClass BasicNameClass(int context) :
+{
+  Annotations a;
+  ParsedNameClass nc;
+}
+{
+  a = Annotations()
+  (nc = PrimaryNameClass(context, a)
+   | nc = OpenNameClass(context, a))
+  { return nc; }
+}
+
+ParsedNameClass PrimaryNameClass(int context, Annotations a) :
+{
+  ParsedNameClass nc;
+}
+{
+  (nc = UnprefixedNameClass(context, a)
+   | nc = PrefixedNameClass(a)
+   | nc = ParenNameClass(context, a))
+  { return nc; }
+}
+
+ParsedNameClass OpenNameClass(int context, Annotations a) :
+{
+  Token t;
+  LocatedString ns;
+}
+{
+  ns = NsName() { checkNsName(context, ns); return ncb.makeNsName(ns.getString(), ns.getLocation(), a); }
+  | t = "*" { checkAnyName(context, t); return ncb.makeAnyName(makeLocation(t), a); }
+}
+
+
+ParsedNameClass UnprefixedNameClass(int context, Annotations a) :
+{
+  LocatedString name;
+}
+{
+  name = UnprefixedName()
+  {
+    String ns;
+    if ((context & (IN_ATTRIBUTE|IN_ELEMENT)) == IN_ATTRIBUTE)
+      ns = "";
+    else
+      ns = defaultNamespace;
+    return ncb.makeName(ns, name.getString(), null, name.getLocation(), a);
+  }
+}
+
+ParsedNameClass PrefixedNameClass(Annotations a) :
+{
+  Token t;
+}
+{
+  t = <PREFIXED_NAME>
+  {
+    String qn = t.image;
+    int colon = qn.indexOf(':');
+    String prefix = qn.substring(0, colon);
+    return ncb.makeName(lookupPrefix(prefix, t), qn.substring(colon + 1), prefix, makeLocation(t), a);
+  }
+}
+
+ParsedNameClass NsNameExceptClass(int context, Annotations a, Annotations[] pa) :
+{
+  LocatedString ns;
+  ParsedNameClass nc;
+}
+{
+  ns = NsName()
+  { checkNsName(context, ns); }
+  (nc = ExceptNameClass(context | IN_NS_NAME)
+   { nc = ncb.makeNsName(ns.getString(), nc, ns.getLocation(), a); }
+   nc = AnnotateAfter(nc)
+  | { nc = ncb.makeNsName(ns.getString(), ns.getLocation(), a); }
+    nc = AnnotateAfter(nc)
+    nc = NameClassAlternatives(context, nc, pa))
+  { return nc; }
+}
+
+LocatedString NsName() :
+{
+  Token t;
+}
+{
+  t = <PREFIX_STAR>
+  {
+    String qn = t.image;
+    String prefix = qn.substring(0, qn.length() - 2);
+    return new LocatedString(lookupPrefix(prefix, t), t);
+  }
+}
+
+ParsedNameClass AnyNameExceptClass(int context, Annotations a, Annotations[] pa) :
+{
+  Token t;
+  ParsedNameClass nc;
+}
+{
+  t = "*"
+  { checkAnyName(context, t); }
+  (nc = ExceptNameClass(context | IN_ANY_NAME)
+   { nc = ncb.makeAnyName(nc, makeLocation(t), a); }
+   nc = AnnotateAfter(nc)
+  | { nc = ncb.makeAnyName(makeLocation(t), a); }
+    nc = AnnotateAfter(nc)
+    nc = NameClassAlternatives(context, nc, pa))
+  { return nc; }
+}
+
+ParsedNameClass ParenNameClass(int context, Annotations a) :
+{
+  Token t;
+  ParsedNameClass nc;
+  Annotations[] pa = new Annotations[]{ a };
+}
+{
+  t = "(" nc = NameClass(context, pa) { nc = afterComments(nc); } ")"
+  {
+    if (pa[0] != null)
+      nc = ncb.makeChoice(Collections.singletonList(nc), makeLocation(t), pa[0]);
+    return nc;
+  }
+}
+
+ParsedNameClass ExceptNameClass(int context) :
+{
+  ParsedNameClass nc;
+}
+{
+  "-" nc = BasicNameClass(context)
+  { return nc; }
+}
+
+ParsedPattern ListExpr(Scope scope, Annotations a) :
+{
+  Token t;
+  ParsedPattern p;
+}
+{
+  t = "list"
+  "{"
+  p = Expr(false, scope, null, null)
+  { p = afterComments(p); }
+  "}"
+  { return sb.makeList(p, makeLocation(t), a); }
+}
+
+ParsedPattern MixedExpr(Scope scope, Annotations a) :
+{
+  Token t;
+  ParsedPattern p;
+}
+{
+  t = "mixed"
+  "{"
+  p = Expr(false, scope, null, null)
+  { p = afterComments(p); }
+  "}"
+  { return sb.makeMixed(p, makeLocation(t), a); }
+}
+
+ParsedPattern GrammarExpr(Scope scope, Annotations a) :
+{
+  Token t;
+  Grammar g;
+}
+{
+  t = "grammar" { g = sb.makeGrammar(scope); }
+  "{" a = GrammarBody(g, g, a) { topLevelComments(g); } "}"
+  { return g.endGrammar(makeLocation(t), a); }
+}
+
+ParsedPattern ParenExpr(boolean topLevel, Scope scope, Annotations a) :
+{
+  Token t;
+  ParsedPattern p;
+}
+{
+  t = "(" p = Expr(topLevel, scope, t, a) { p = afterComments(p); } ")"
+  { return p; }
+}
+
+Annotations GrammarBody(GrammarSection section, Scope scope, Annotations a) :
+{
+  ParsedElementAnnotation e;
+}
+{
+  (LOOKAHEAD(2) e = AnnotationElementNotKeyword()
+   { if (a == null) a = sb.makeAnnotations(null, getContext()); a.addElement(e); })*
+  (GrammarComponent(section, scope))*
+  { return a; }
+}
+
+void GrammarComponent(GrammarSection section, Scope scope) :
+{
+  ParsedElementAnnotation e;
+  Annotations a;
+}
+{
+  (a = Annotations()
+   (Definition(section, scope, a)
+    | Include(section, scope, a)
+    | Div(section, scope, a)))
+  (LOOKAHEAD(2) e = AnnotationElementNotKeyword() { section.topLevelAnnotation(e); })*
+}
+
+void Definition(GrammarSection section, Scope scope, Annotations a) :
+{}
+{
+  (Define(section, scope, a) | Start(section, scope, a))
+}
+
+void Start(GrammarSection section, Scope scope, Annotations a) :
+{
+  Token t;
+  GrammarSection.Combine combine;
+  ParsedPattern p;
+}
+{
+  t = "start" combine = AssignOp() p = Expr(false, scope, null, null)
+  { section.define(GrammarSection.START, combine, p, makeLocation(t), a); }
+}
+
+void Define(GrammarSection section, Scope scope, Annotations a) :
+{
+  LocatedString name;
+  GrammarSection.Combine combine;
+  ParsedPattern p;
+}
+{
+  name = Identifier() combine = AssignOp() p = Expr(false, scope, null, null)
+  { section.define(name.getString(), combine, p, name.getLocation(), a); }
+}
+
+GrammarSection.Combine AssignOp() :
+{}
+{
+  "=" { return null; }
+  | "|=" { return GrammarSection.COMBINE_CHOICE; }
+  | "&=" { return GrammarSection.COMBINE_INTERLEAVE; }
+}
+
+void Include(GrammarSection section, Scope scope, Annotations a) :
+{
+  Token t;
+  String href;
+  String ns;
+  Include include = section.makeInclude();
+}
+{
+  t = "include" href = Literal()
+  ns = Inherit()
+  ("{" a = IncludeBody(include, scope, a) { topLevelComments(include); } "}")?
+  {
+    try {
+      include.endInclude(parseable, resolve(href), ns, makeLocation(t), a);
+    }
+    catch (IllegalSchemaException e) { }
+  }
+}
+
+Annotations IncludeBody(GrammarSection section, Scope scope, Annotations a) :
+{
+  ParsedElementAnnotation e;
+}
+{
+  (LOOKAHEAD(2) e = AnnotationElementNotKeyword()
+   { if (a == null) a = sb.makeAnnotations(null, getContext()); a.addElement(e); })*
+  (IncludeComponent(section, scope))*
+  { return a; }
+}
+
+
+void IncludeComponent(GrammarSection section, Scope scope) :
+{
+  ParsedElementAnnotation e;
+  Annotations a;
+}
+{
+  (a = Annotations() (Definition(section, scope, a)
+                      | IncludeDiv(section, scope, a)))
+  (LOOKAHEAD(2) e = AnnotationElementNotKeyword() { section.topLevelAnnotation(e); })*
+}
+
+void Div(GrammarSection section, Scope scope, Annotations a) :
+{
+  Token t;
+  Div div = section.makeDiv();
+}
+{
+  t = "div" "{" a = GrammarBody(div, scope, a) { topLevelComments(div); } "}"
+  { div.endDiv(makeLocation(t), a); }
+}
+
+void IncludeDiv(GrammarSection section, Scope scope, Annotations a) :
+{
+  Token t;
+  Div div = section.makeDiv();
+}
+{
+  t = "div" "{" a = IncludeBody(div, scope, a) { topLevelComments(div); } "}"
+  { div.endDiv(makeLocation(t), a); }
+}
+
+ParsedPattern ExternalRefExpr(Scope scope, Annotations a) :
+{
+  Token t;
+  String href;
+  String ns;
+}
+{
+  t = "external" href = Literal()
+  ns = Inherit()
+  {
+    try {
+      return sb.makeExternalRef(parseable, resolve(href), ns, scope, makeLocation(t), a);
+    }
+    catch (IllegalSchemaException e) {
+      return sb.makeErrorPattern();
+    }
+  }
+}
+
+String Inherit() :
+{
+  String ns = null;
+}
+{
+  ("inherit" "=" ns = Prefix())?
+  {
+    if (ns == null)
+      ns = defaultNamespace;
+    return ns;
+  }
+}
+
+ParsedPattern ParentExpr(Scope scope, Annotations a) :
+{
+  LocatedString name;
+}
+{
+  "parent" { a = addCommentsToChildAnnotations(a); } name = Identifier()
+  {
+    if(scope==null) {
+      error("parent_ref_outside_grammar",name.getToken());
+      return sb.makeErrorPattern();
+    } else {
+      return scope.makeParentRef(name.getString(), name.getLocation(), a);
+    }
+  }
+}
+
+ParsedPattern IdentifierExpr(Scope scope, Annotations a) :
+{
+  LocatedString name;
+}
+{
+  name = Identifier()
+  {
+    if(scope==null) {
+      error("ref_outside_grammar",name.getToken());
+      return sb.makeErrorPattern();
+    } else {
+      return scope.makeRef(name.getString(), name.getLocation(), a);
+    }
+  }
+}
+
+ParsedPattern ValueExpr(boolean topLevel, Annotations a) :
+{
+  LocatedString s;
+}
+{
+  s = LocatedLiteral()
+  {
+    if (topLevel && annotationsIncludeElements) {
+      error("top_level_follow_annotation", s.getToken());
+      a = null;
+    }
+    return sb.makeValue("", "token", s.getString(), getContext(), defaultNamespace, s.getLocation(), a);
+  }
+}
+
+ParsedPattern DataExpr(boolean topLevel, Scope scope, Annotations a, Token[] except) :
+{
+  Token datatypeToken;
+  Location loc;
+  String datatype;
+  String datatypeUri = null;
+  String s = null;
+  ParsedPattern e = null;
+  DataPatternBuilder dpb;
+}
+{
+  datatypeToken = DatatypeName()
+  {
+    datatype = datatypeToken.image;
+    loc = makeLocation(datatypeToken);
+    int colon = datatype.indexOf(':');
+    if (colon < 0)
+      datatypeUri = "";
+    else {
+      String prefix = datatype.substring(0, colon);
+      datatypeUri = lookupDatatype(prefix, datatypeToken);
+      datatype = datatype.substring(colon + 1);
+    }
+  }
+  ((s = Literal()
+    {
+      if (topLevel && annotationsIncludeElements) {
+        error("top_level_follow_annotation", datatypeToken);
+        a = null;
+      }
+      return sb.makeValue(datatypeUri, datatype, s, getContext(), defaultNamespace, loc, a);
+    }
+  )
+  | ( { dpb = sb.makeDataPatternBuilder(datatypeUri, datatype, loc); }
+      ( (Params(dpb) (e = Except(scope, except))?)
+        | (e = Except(scope, except))?)
+      { return e == null ? dpb.makePattern(loc, a) : dpb.makePattern(e, loc, a); }))
+}
+
+Token DatatypeName() :
+{
+  Token t;
+}
+{
+  (t = "string" | t = "token" | t = <PREFIXED_NAME>)
+  { return t; }
+}
+
+LocatedString Identifier() :
+{
+  LocatedString s;
+  Token t;
+}
+{
+  (t = <IDENTIFIER> { s = new LocatedString(t.image, t); }
+  | t = <ESCAPED_IDENTIFIER> { s = new LocatedString(t.image.substring(1), t); })
+  { return s; }
+}
+
+String Prefix() :
+{
+  Token t;
+  String prefix;
+}
+{
+  (t = <IDENTIFIER> { prefix = t.image; }
+  | t = <ESCAPED_IDENTIFIER> { prefix = t.image.substring(1); }
+  | t = Keyword() { prefix = t.image; })
+  { return lookupPrefix(prefix, t); }
+}
+
+LocatedString UnprefixedName() :
+{
+  LocatedString s;
+  Token t;
+}
+{
+  (s = Identifier()
+   | t = Keyword() { s = new LocatedString(t.image, t); })
+  { return s; }
+}
+
+void Params(DataPatternBuilder dpb) :
+{}
+{
+  "{" (Param(dpb))* "}"
+}
+
+void Param(DataPatternBuilder dpb) :
+{
+  LocatedString name;
+  Annotations a;
+  String value;
+}
+{
+  a = Annotations() name = UnprefixedName() "=" { a = addCommentsToLeadingAnnotations(a); } value = Literal()
+  { dpb.addParam(name.getString(), value, getContext(), defaultNamespace, name.getLocation(), a); }
+}
+
+ParsedPattern Except(Scope scope, Token[] except) :
+{
+  Annotations a;
+  ParsedPattern p;
+  Token t;
+  Token[] innerExcept = new Token[1];
+}
+{
+  t = "-" a = Annotations() p = PrimaryExpr(false, scope, a, innerExcept)
+  {
+    checkExcept(innerExcept);
+    except[0] = t;
+    return p;
+  }
+}
+
+ParsedElementAnnotation Documentation() :
+{
+  CommentList comments = getComments();
+  ElementAnnotationBuilder eab;
+  Token t;
+}
+{
+  (t = <DOCUMENTATION> | t = <DOCUMENTATION_AFTER_SINGLE_LINE_COMMENT>)
+  {
+    eab = sb.makeElementAnnotationBuilder(WellKnownNamespaces.RELAX_NG_COMPATIBILITY_ANNOTATIONS,
+                                          "documentation",
+                                          getCompatibilityPrefix(),
+                                          makeLocation(t),
+                                          comments,
+                                          getContext());
+    eab.addText(mungeComment(t.image), makeLocation(t), null);
+  }
+  (t = <DOCUMENTATION_CONTINUE> { eab.addText("\n" + mungeComment(t.image), makeLocation(t), null); })*
+  { return eab.makeElementAnnotation(); }
+}
+
+Annotations Annotations() :
+{
+  CommentList comments = getComments();
+  Annotations a = null;
+  ParsedElementAnnotation e;
+}
+{
+  ( { a = sb.makeAnnotations(comments, getContext()); }
+    (e = Documentation() { a.addElement(e); })+
+    {
+      comments = getComments();
+      if (comments != null)
+        a.addLeadingComment(comments);
+    }
+  )?
+  ("[" { if (a == null) a = sb.makeAnnotations(comments, getContext()); clearAttributeList(); annotationsIncludeElements = false; }
+       (LOOKAHEAD(2) PrefixedAnnotationAttribute(a, false) )*
+       ( e = AnnotationElement(false) { a.addElement(e); annotationsIncludeElements = true; } )*
+       { a.addComment(getComments()); }
+  "]")?
+  {
+    if (a == null && comments != null)
+      a = sb.makeAnnotations(comments, getContext());
+    return a;
+  }
+}
+
+void AnnotationAttribute(Annotations a) :
+{}
+{
+  PrefixedAnnotationAttribute(a, true) | UnprefixedAnnotationAttribute(a)
+}
+
+void PrefixedAnnotationAttribute(Annotations a, boolean nested) :
+{
+  Token t;
+  String value;
+}
+{
+  t = <PREFIXED_NAME> "=" value = Literal()
+  {
+    String qn = t.image;
+    int colon = qn.indexOf(':');
+    String prefix = qn.substring(0, colon);
+    String ns = lookupPrefix(prefix, t);
+    if (ns == this.inheritedNs)
+      error("inherited_annotation_namespace", t);
+    else if (ns.length() == 0 && !nested)
+      error("unqualified_annotation_attribute", t);
+    else if (ns.equals(WellKnownNamespaces.RELAX_NG) && !nested)
+      error("relax_ng_namespace", t);
+    /*else if (ns.length() == 0
+             && qn.length() - colon - 1 == 5
+             && qn.regionMatches(colon + 1, "xmlns", 0, 5))
+      error("xmlns_annotation_attribute", t);*/
+    else if (ns.equals(WellKnownNamespaces.XMLNS))
+      error("xmlns_annotation_attribute_uri", t);
+    else {
+      if (ns.length() == 0)
+	prefix = null;
+      addAttribute(a, ns, qn.substring(colon + 1), prefix, value, t);
+    }
+  }
+}
+
+void UnprefixedAnnotationAttribute(Annotations a) :
+{
+  LocatedString name;
+  String value;
+}
+{
+  name = UnprefixedName() "=" value = Literal()
+  {
+    if (name.getString().equals("xmlns"))
+      error("xmlns_annotation_attribute", name.getToken());
+    else
+      addAttribute(a, "", name.getString(), null, value, name.getToken());
+  }
+}
+
+ParsedElementAnnotation AnnotationElement(boolean nested) :
+{
+  ParsedElementAnnotation a;
+}
+{
+  (a = PrefixedAnnotationElement(nested)
+   | a = UnprefixedAnnotationElement())
+  { return a; }
+}
+
+ParsedElementAnnotation AnnotationElementNotKeyword() :
+{
+  ParsedElementAnnotation a;
+}
+{
+  (a = PrefixedAnnotationElement(false)
+   | a = IdentifierAnnotationElement())
+  { return a; }
+}
+
+ParsedElementAnnotation PrefixedAnnotationElement(boolean nested) :
+{
+  CommentList comments = getComments();
+  Token t;
+  ElementAnnotationBuilder eab;
+}
+{
+  t = <PREFIXED_NAME>
+  {
+    String qn = t.image;
+    int colon = qn.indexOf(':');
+    String prefix = qn.substring(0, colon);
+    String ns = lookupPrefix(prefix, t);
+    if (ns == this.inheritedNs) {
+      error("inherited_annotation_namespace", t);
+      ns = "";
+    }
+    else if (!nested && ns.equals(WellKnownNamespaces.RELAX_NG)) {
+      error("relax_ng_namespace", t);
+      ns = "";
+    }
+    else {
+      if (ns.length() == 0)
+        prefix = null;
+    }
+    eab = sb.makeElementAnnotationBuilder(ns, qn.substring(colon + 1), prefix,
+					  makeLocation(t), comments, getContext());
+  }
+  AnnotationElementContent(eab)
+  { return eab.makeElementAnnotation(); }
+}
+
+ParsedElementAnnotation UnprefixedAnnotationElement() :
+{
+  CommentList comments = getComments();
+  LocatedString name;
+  ElementAnnotationBuilder eab;
+}
+{
+  name = UnprefixedName()
+  {
+    eab = sb.makeElementAnnotationBuilder("", name.getString(), null,
+                                          name.getLocation(), comments, getContext());
+  }
+  AnnotationElementContent(eab)
+  { return eab.makeElementAnnotation(); }
+}
+
+ParsedElementAnnotation IdentifierAnnotationElement() :
+{
+  CommentList comments = getComments();
+  LocatedString name;
+  ElementAnnotationBuilder eab;
+}
+{
+  name = Identifier()
+  {
+    eab = sb.makeElementAnnotationBuilder("", name.getString(), null,
+                                          name.getLocation(), comments, getContext());
+  }
+  AnnotationElementContent(eab)
+  { return eab.makeElementAnnotation(); }
+}
+
+void AnnotationElementContent(ElementAnnotationBuilder eab) :
+{
+  ParsedElementAnnotation e;
+}
+{
+  "[" { clearAttributeList(); }
+  (LOOKAHEAD(2) AnnotationAttribute(eab))*
+  ((AnnotationElementLiteral(eab)
+    ("~" AnnotationElementLiteral(eab))*)
+   | e = AnnotationElement(true) { eab.addElement(e); })*
+  { eab.addComment(getComments()); }
+  "]"
+}
+
+void AnnotationElementLiteral(ElementAnnotationBuilder eab) :
+{
+  Token t;
+  CommentList comments = getComments();
+}
+{
+  t = <LITERAL> { eab.addText(unquote(t.image), makeLocation(t), comments); }
+}
+
+String Literal() :
+{
+  Token t;
+  String s;
+  StringBuffer buf;
+}
+{
+  t = <LITERAL>
+  {
+    s = unquote(t.image);
+  }
+  (
+    { buf = new StringBuffer(s); }
+    ("~" t = <LITERAL> { buf.append(unquote(t.image)); })+
+    { s = buf.toString(); }
+  )?
+  { return s; }
+}
+
+LocatedString LocatedLiteral() :
+{
+  Token t;
+  Token t2;
+  String s;
+  StringBuffer buf;
+}
+{
+  t = <LITERAL>
+  {
+    s = unquote(t.image);
+  }
+  (
+    { buf = new StringBuffer(s); }
+    ("~" t2 = <LITERAL> { buf.append(unquote(t2.image)); })+
+    { s = buf.toString(); }
+  )?
+  { return new LocatedString(s, t); }
+}
+
+Token Keyword() :
+{
+  Token t;
+}
+{
+  (t = "element"
+  | t = "attribute"
+  | t = "namespace"
+  | t = "list"
+  | t = "mixed"
+  | t = "grammar"
+  | t = "empty"
+  | t = "text"
+  | t = "parent"
+  | t = "external"
+  | t = "notAllowed"
+  | t = "start"
+  | t = "include"
+  | t = "default"
+  | t = "inherit"
+  | t = "string"
+  | t = "token"
+  | t = "datatypes"
+  | t = "div")
+  { return t; }
+}
+
+<*>
+SKIP: {
+  < #NEWLINE : [ "\u0000", "\n" ] >
+  | < #NOT_NEWLINE : ~[ "\u0000", "\n" ] >
+  | < WS: ([ "\u0000", " ", "\n", "\t" ])+ > : DEFAULT
+}
+
+TOKEN :
+{
+  < DOCUMENTATION: "##" (<NOT_NEWLINE>)* > : AFTER_DOCUMENTATION
+}
+
+<AFTER_DOCUMENTATION>
+TOKEN :
+{
+  < DOCUMENTATION_CONTINUE: <NEWLINE> ([" ", "\t"])* <DOCUMENTATION> >
+}
+
+SPECIAL_TOKEN:
+{
+  < SINGLE_LINE_COMMENT: "#" (<NOT_NEWLINE>)* > : AFTER_SINGLE_LINE_COMMENT
+}
+
+<AFTER_SINGLE_LINE_COMMENT>
+TOKEN :
+{
+  < DOCUMENTATION_AFTER_SINGLE_LINE_COMMENT: <NEWLINE> ([" ", "\t"])* <DOCUMENTATION> > : AFTER_DOCUMENTATION
+}
+
+<AFTER_SINGLE_LINE_COMMENT>
+SPECIAL_TOKEN :
+{
+  < SINGLE_LINE_COMMENT_CONTINUE: <NEWLINE> ([" ", "\t"])* <SINGLE_LINE_COMMENT> >
+}
+
+TOKEN :
+{
+  < #BASE_CHAR : [
+    "\u0041" - "\u005a",
+    "\u0061" - "\u007a",
+    "\u00c0" - "\u00d6",
+    "\u00d8" - "\u00f6",
+    "\u00f8" - "\u00ff",
+    "\u0100" - "\u0131",
+    "\u0134" - "\u013e",
+    "\u0141" - "\u0148",
+    "\u014a" - "\u017e",
+    "\u0180" - "\u01c3",
+    "\u01cd" - "\u01f0",
+    "\u01f4" - "\u01f5",
+    "\u01fa" - "\u0217",
+    "\u0250" - "\u02a8",
+    "\u02bb" - "\u02c1",
+    "\u0386",
+    "\u0388" - "\u038a",
+    "\u038c",
+    "\u038e" - "\u03a1",
+    "\u03a3" - "\u03ce",
+    "\u03d0" - "\u03d6",
+    "\u03da",
+    "\u03dc",
+    "\u03de",
+    "\u03e0",
+    "\u03e2" - "\u03f3",
+    "\u0401" - "\u040c",
+    "\u040e" - "\u044f",
+    "\u0451" - "\u045c",
+    "\u045e" - "\u0481",
+    "\u0490" - "\u04c4",
+    "\u04c7" - "\u04c8",
+    "\u04cb" - "\u04cc",
+    "\u04d0" - "\u04eb",
+    "\u04ee" - "\u04f5",
+    "\u04f8" - "\u04f9",
+    "\u0531" - "\u0556",
+    "\u0559",
+    "\u0561" - "\u0586",
+    "\u05d0" - "\u05ea",
+    "\u05f0" - "\u05f2",
+    "\u0621" - "\u063a",
+    "\u0641" - "\u064a",
+    "\u0671" - "\u06b7",
+    "\u06ba" - "\u06be",
+    "\u06c0" - "\u06ce",
+    "\u06d0" - "\u06d3",
+    "\u06d5",
+    "\u06e5" - "\u06e6",
+    "\u0905" - "\u0939",
+    "\u093d",
+    "\u0958" - "\u0961",
+    "\u0985" - "\u098c",
+    "\u098f" - "\u0990",
+    "\u0993" - "\u09a8",
+    "\u09aa" - "\u09b0",
+    "\u09b2",
+    "\u09b6" - "\u09b9",
+    "\u09dc" - "\u09dd",
+    "\u09df" - "\u09e1",
+    "\u09f0" - "\u09f1",
+    "\u0a05" - "\u0a0a",
+    "\u0a0f" - "\u0a10",
+    "\u0a13" - "\u0a28",
+    "\u0a2a" - "\u0a30",
+    "\u0a32" - "\u0a33",
+    "\u0a35" - "\u0a36",
+    "\u0a38" - "\u0a39",
+    "\u0a59" - "\u0a5c",
+    "\u0a5e",
+    "\u0a72" - "\u0a74",
+    "\u0a85" - "\u0a8b",
+    "\u0a8d",
+    "\u0a8f" - "\u0a91",
+    "\u0a93" - "\u0aa8",
+    "\u0aaa" - "\u0ab0",
+    "\u0ab2" - "\u0ab3",
+    "\u0ab5" - "\u0ab9",
+    "\u0abd",
+    "\u0ae0",
+    "\u0b05" - "\u0b0c",
+    "\u0b0f" - "\u0b10",
+    "\u0b13" - "\u0b28",
+    "\u0b2a" - "\u0b30",
+    "\u0b32" - "\u0b33",
+    "\u0b36" - "\u0b39",
+    "\u0b3d",
+    "\u0b5c" - "\u0b5d",
+    "\u0b5f" - "\u0b61",
+    "\u0b85" - "\u0b8a",
+    "\u0b8e" - "\u0b90",
+    "\u0b92" - "\u0b95",
+    "\u0b99" - "\u0b9a",
+    "\u0b9c",
+    "\u0b9e" - "\u0b9f",
+    "\u0ba3" - "\u0ba4",
+    "\u0ba8" - "\u0baa",
+    "\u0bae" - "\u0bb5",
+    "\u0bb7" - "\u0bb9",
+    "\u0c05" - "\u0c0c",
+    "\u0c0e" - "\u0c10",
+    "\u0c12" - "\u0c28",
+    "\u0c2a" - "\u0c33",
+    "\u0c35" - "\u0c39",
+    "\u0c60" - "\u0c61",
+    "\u0c85" - "\u0c8c",
+    "\u0c8e" - "\u0c90",
+    "\u0c92" - "\u0ca8",
+    "\u0caa" - "\u0cb3",
+    "\u0cb5" - "\u0cb9",
+    "\u0cde",
+    "\u0ce0" - "\u0ce1",
+    "\u0d05" - "\u0d0c",
+    "\u0d0e" - "\u0d10",
+    "\u0d12" - "\u0d28",
+    "\u0d2a" - "\u0d39",
+    "\u0d60" - "\u0d61",
+    "\u0e01" - "\u0e2e",
+    "\u0e30",
+    "\u0e32" - "\u0e33",
+    "\u0e40" - "\u0e45",
+    "\u0e81" - "\u0e82",
+    "\u0e84",
+    "\u0e87" - "\u0e88",
+    "\u0e8a",
+    "\u0e8d",
+    "\u0e94" - "\u0e97",
+    "\u0e99" - "\u0e9f",
+    "\u0ea1" - "\u0ea3",
+    "\u0ea5",
+    "\u0ea7",
+    "\u0eaa" - "\u0eab",
+    "\u0ead" - "\u0eae",
+    "\u0eb0",
+    "\u0eb2" - "\u0eb3",
+    "\u0ebd",
+    "\u0ec0" - "\u0ec4",
+    "\u0f40" - "\u0f47",
+    "\u0f49" - "\u0f69",
+    "\u10a0" - "\u10c5",
+    "\u10d0" - "\u10f6",
+    "\u1100",
+    "\u1102" - "\u1103",
+    "\u1105" - "\u1107",
+    "\u1109",
+    "\u110b" - "\u110c",
+    "\u110e" - "\u1112",
+    "\u113c",
+    "\u113e",
+    "\u1140",
+    "\u114c",
+    "\u114e",
+    "\u1150",
+    "\u1154" - "\u1155",
+    "\u1159",
+    "\u115f" - "\u1161",
+    "\u1163",
+    "\u1165",
+    "\u1167",
+    "\u1169",
+    "\u116d" - "\u116e",
+    "\u1172" - "\u1173",
+    "\u1175",
+    "\u119e",
+    "\u11a8",
+    "\u11ab",
+    "\u11ae" - "\u11af",
+    "\u11b7" - "\u11b8",
+    "\u11ba",
+    "\u11bc" - "\u11c2",
+    "\u11eb",
+    "\u11f0",
+    "\u11f9",
+    "\u1e00" - "\u1e9b",
+    "\u1ea0" - "\u1ef9",
+    "\u1f00" - "\u1f15",
+    "\u1f18" - "\u1f1d",
+    "\u1f20" - "\u1f45",
+    "\u1f48" - "\u1f4d",
+    "\u1f50" - "\u1f57",
+    "\u1f59",
+    "\u1f5b",
+    "\u1f5d",
+    "\u1f5f" - "\u1f7d",
+    "\u1f80" - "\u1fb4",
+    "\u1fb6" - "\u1fbc",
+    "\u1fbe",
+    "\u1fc2" - "\u1fc4",
+    "\u1fc6" - "\u1fcc",
+    "\u1fd0" - "\u1fd3",
+    "\u1fd6" - "\u1fdb",
+    "\u1fe0" - "\u1fec",
+    "\u1ff2" - "\u1ff4",
+    "\u1ff6" - "\u1ffc",
+    "\u2126",
+    "\u212a" - "\u212b",
+    "\u212e",
+    "\u2180" - "\u2182",
+    "\u3041" - "\u3094",
+    "\u30a1" - "\u30fa",
+    "\u3105" - "\u312c",
+    "\uac00" - "\ud7a3"
+    ] >
+  | < #IDEOGRAPHIC : [
+    "\u4e00" - "\u9fa5",
+    "\u3007",
+    "\u3021" - "\u3029"
+    ] >
+  | < #LETTER : (<BASE_CHAR> | <IDEOGRAPHIC>) >
+  | < #COMBINING_CHAR : [
+    "\u0300" - "\u0345",
+    "\u0360" - "\u0361",
+    "\u0483" - "\u0486",
+    "\u0591" - "\u05a1",
+    "\u05a3" - "\u05b9",
+    "\u05bb" - "\u05bd",
+    "\u05bf",
+    "\u05c1" - "\u05c2",
+    "\u05c4",
+    "\u064b" - "\u0652",
+    "\u0670",
+    "\u06d6" - "\u06dc",
+    "\u06dd" - "\u06df",
+    "\u06e0" - "\u06e4",
+    "\u06e7" - "\u06e8",
+    "\u06ea" - "\u06ed",
+    "\u0901" - "\u0903",
+    "\u093c",
+    "\u093e" - "\u094c",
+    "\u094d",
+    "\u0951" - "\u0954",
+    "\u0962" - "\u0963",
+    "\u0981" - "\u0983",
+    "\u09bc",
+    "\u09be",
+    "\u09bf",
+    "\u09c0" - "\u09c4",
+    "\u09c7" - "\u09c8",
+    "\u09cb" - "\u09cd",
+    "\u09d7",
+    "\u09e2" - "\u09e3",
+    "\u0a02",
+    "\u0a3c",
+    "\u0a3e",
+    "\u0a3f",
+    "\u0a40" - "\u0a42",
+    "\u0a47" - "\u0a48",
+    "\u0a4b" - "\u0a4d",
+    "\u0a70" - "\u0a71",
+    "\u0a81" - "\u0a83",
+    "\u0abc",
+    "\u0abe" - "\u0ac5",
+    "\u0ac7" - "\u0ac9",
+    "\u0acb" - "\u0acd",
+    "\u0b01" - "\u0b03",
+    "\u0b3c",
+    "\u0b3e" - "\u0b43",
+    "\u0b47" - "\u0b48",
+    "\u0b4b" - "\u0b4d",
+    "\u0b56" - "\u0b57",
+    "\u0b82" - "\u0b83",
+    "\u0bbe" - "\u0bc2",
+    "\u0bc6" - "\u0bc8",
+    "\u0bca" - "\u0bcd",
+    "\u0bd7",
+    "\u0c01" - "\u0c03",
+    "\u0c3e" - "\u0c44",
+    "\u0c46" - "\u0c48",
+    "\u0c4a" - "\u0c4d",
+    "\u0c55" - "\u0c56",
+    "\u0c82" - "\u0c83",
+    "\u0cbe" - "\u0cc4",
+    "\u0cc6" - "\u0cc8",
+    "\u0cca" - "\u0ccd",
+    "\u0cd5" - "\u0cd6",
+    "\u0d02" - "\u0d03",
+    "\u0d3e" - "\u0d43",
+    "\u0d46" - "\u0d48",
+    "\u0d4a" - "\u0d4d",
+    "\u0d57",
+    "\u0e31",
+    "\u0e34" - "\u0e3a",
+    "\u0e47" - "\u0e4e",
+    "\u0eb1",
+    "\u0eb4" - "\u0eb9",
+    "\u0ebb" - "\u0ebc",
+    "\u0ec8" - "\u0ecd",
+    "\u0f18" - "\u0f19",
+    "\u0f35",
+    "\u0f37",
+    "\u0f39",
+    "\u0f3e",
+    "\u0f3f",
+    "\u0f71" - "\u0f84",
+    "\u0f86" - "\u0f8b",
+    "\u0f90" - "\u0f95",
+    "\u0f97",
+    "\u0f99" - "\u0fad",
+    "\u0fb1" - "\u0fb7",
+    "\u0fb9",
+    "\u20d0" - "\u20dc",
+    "\u20e1",
+    "\u302a" - "\u302f",
+    "\u3099",
+    "\u309a"
+    ] >
+  | < #DIGIT : [
+    "\u0030" - "\u0039",
+    "\u0660" - "\u0669",
+    "\u06f0" - "\u06f9",
+    "\u0966" - "\u096f",
+    "\u09e6" - "\u09ef",
+    "\u0a66" - "\u0a6f",
+    "\u0ae6" - "\u0aef",
+    "\u0b66" - "\u0b6f",
+    "\u0be7" - "\u0bef",
+    "\u0c66" - "\u0c6f",
+    "\u0ce6" - "\u0cef",
+    "\u0d66" - "\u0d6f",
+    "\u0e50" - "\u0e59",
+    "\u0ed0" - "\u0ed9",
+    "\u0f20" - "\u0f29"
+    ] >
+  | < #EXTENDER : [
+    "\u00b7",
+    "\u02d0",
+    "\u02d1",
+    "\u0387",
+    "\u0640",
+    "\u0e46",
+    "\u0ec6",
+    "\u3005",
+    "\u3031" - "\u3035",
+    "\u309d" - "\u309e",
+    "\u30fc" - "\u30fe"
+    ] >
+  | < #NMSTART : (<LETTER> | "_") >
+  | < #NMCHAR : (<LETTER> | <COMBINING_CHAR> | <EXTENDER> | <DIGIT> | "." | "-" | "_") >
+  | < #NCNAME: <NMSTART> (<NMCHAR>)* >
+}
+
+TOKEN :
+{
+  < IDENTIFIER: <NCNAME> >
+  | < ESCAPED_IDENTIFIER: "\\" <NCNAME> >
+  | < PREFIX_STAR: <NCNAME> ":*" >
+  | < PREFIXED_NAME: <NCNAME> ":" <NCNAME> >
+  | < LITERAL : ("\"" (~["\u0000", "\""])* "\"")
+                | ("'" (~["\u0000", "'"])* "'")
+                | ("\"\"\"" (~["\""]
+                             | ("\"" ~["\""])
+                             | ("\"\"" ~["\""]))* "\"\"\"")
+                | ("'''" (~["'"]
+                             | ("'" ~["'"])
+                             | ("''" ~["'"]))* "'''") >
+  | < FANNOTATE : ">>" >
+}
+
+/* This avoids lexical errors from JavaCC. */
+<*>
+TOKEN :
+{
+  < ILLEGAL_CHAR : [ "\u0000" - "\u0008", "\u000b" - "\uffff" ] >
+}