+ * Provide a hand-written scanner for the Java language. + */ + +public class CScanner extends Scanner { + + private boolean debug = false; + + /** Create a Java scanner, for Java version 1.5 by default. */ + public CScanner() { + super(); + initKind(); + initUniKind(); + } + + /** Create a Java scanner, for a given version between "1.1" and "1.5". */ + public CScanner(String version) { + super(); + initKind(); + initUniKind(); + } + + /** Override the read method from the Scanner class. */ + protected int read() { + int type, saveStart = 0; + if (debug) + saveStart = start; + + if (start >= end) + return WHITESPACE; + + switch (state) { + case MID_COMMENT: + case END_COMMENT: + type = readComment(MID_COMMENT); + if (type == END_COMMENT) + state = WHITESPACE; + else + state = MID_COMMENT; + return type; + default: + char c = buffer[start]; + if (c == '\\') + c = next(); + if (c < 128) + type = kind[c]; + else + type = unikind[Character.getType(c)]; + switch (type) { + case WHITESPACE: + start = start + charlength; + charlength = 1; + while (start < end) { + c = buffer[start]; + if (c == '\\') + c = next(); + int k; + if (c < 128) + k = kind[c]; + else + k = unikind[Character.getType(c)]; + if (k != WHITESPACE) + break; + start = start + charlength; + charlength = 1; + } + break; + case UNRECOGNIZED: + case BRACKET: + case SEPARATOR: + start = start + charlength; + charlength = 1; + break; + case OPERATOR: + start = start + charlength; + charlength = 1; + type = readOperator(c); + break; + case CHARACTER: + start = start + charlength; + charlength = 1; + type = readCharLiteral(); + break; + case STRING: + start = start + charlength; + charlength = 1; + type = readStringLiteral(); + break; + case IDENTIFIER: + start = start + charlength; + charlength = 1; + while (start < end) { + c = buffer[start]; + if (c == '\\') + c = next(); + int k; + if (c < 128) + k = kind[c]; + else + k = unikind[Character.getType(c)]; + if (k != IDENTIFIER && k != NUMBER) + break; + start = start + charlength; + charlength = 1; + } + break; + case NUMBER: + start = start + charlength; + charlength = 1; + type = readNumber(c); + break; + case PUNCTUATION: + start = start + charlength; + charlength = 1; + type = readDot(); + break; + case COMMENT: + start = start + charlength; + charlength = 1; + type = readSlash(); + if (type == START_COMMENT) + state = MID_COMMENT; + break; + } + } + if (debug) { + System.out.print(TokenTypes.typeNames[type]); + System.out.println(" " + saveStart + "," + end + "(" + + (start - saveStart) + ")"); + } + return type; + } + + private int readOperator(char c) { + if (start >= end) + return OPERATOR; + char c2; + + switch (c) { + case '~': + case '?': + case ':': + break; + case '+': + case '-': + case '&': + case '|': + c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (c2 != c && c2 != '=') + break; + start = start + charlength; + charlength = 1; + break; + case '=': + case '*': + case '!': + case '^': + case '%': + case '/': + c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (c2 != '=') + break; + start = start + charlength; + charlength = 1; + break; + case '<': + case '>': + c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (c2 == '=') { + start = start + charlength; + charlength = 1; + } else if (c2 == c) { + start = start + charlength; + charlength = 1; + if (start >= end) + break; + char c3 = buffer[start]; + if (c3 == '\\') + c3 = next(); + if (c3 == '=') { + start = start + charlength; + charlength = 1; + } else if (c == '>' && c3 == '>') // >>> + { + start = start + charlength; + charlength = 1; + if (start >= end) + break; + char c4 = buffer[start]; + if (c4 == '\\') + c4 = next(); + if (c4 != '=') + break; + start = start + charlength; + charlength = 1; + } + } + break; + } + return OPERATOR; + } + + private int readCharLiteral() { + if (start >= end) + return bad(CHARACTER); + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + + switch (c2) { + case '\\': + start = start + charlength; + charlength = 1; + boolean ok = readEscapeSequence(); + if (!ok) + return bad(CHARACTER); + break; + case '\'': + case '\n': + return bad(CHARACTER); + default: + start = start + charlength; + charlength = 1; + break; + } + if (start >= end) + return bad(CHARACTER); + char c3 = buffer[start]; + if (c3 == '\\') + c3 = next(); + if (c3 != '\'') + return bad(CHARACTER); + start = start + charlength; + charlength = 1; + return CHARACTER; + } + + private int readStringLiteral() { + if (start >= end) + return bad(STRING); + char c = buffer[start]; + if (c == '\\') + c = next(); + + while (c != '"') { + switch (c) { + case '\\': + start = start + charlength; + charlength = 1; + boolean ok = readEscapeSequence(); + if (!ok) + return bad(STRING); + break; + case '\n': + return bad(STRING); + default: + start = start + charlength; + charlength = 1; + if (start >= end) + return bad(STRING); + break; + } + c = buffer[start]; + if (c == '\\') + c = next(); + } + if (c != '"') + return bad(STRING); + start = start + charlength; + charlength = 1; + return STRING; + } + + private int readSlash() { + if (start >= end) + return OPERATOR; + char c = buffer[start]; + if (c == '\\') + c = next(); + if (c == '/') { + while (c != '\n') { + start = start + charlength; + charlength = 1; + if (start >= end) + return COMMENT; + c = buffer[start]; + if (c == '\\') + c = next(); + } + start = start + charlength; + charlength = 1; + return COMMENT; + } else if (c == '*') { + start = start + charlength; + charlength = 1; + return readComment(START_COMMENT); + } + return readOperator('/'); + } + + // Read one line of a /*...*/ comment, given the expected type + int readComment(int type) { + if (start >= end) + return type; + char c = buffer[start]; + if (c == '\\') + c = next(); + + while (true) { + while (c != '*' && c != '\n') { + start = start + charlength; + charlength = 1; + if (start >= end) + return type; + c = buffer[start]; + if (c == '\\') + c = next(); + } + start = start + charlength; + charlength = 1; + if (c == '\n') + return type; + if (start >= end) + return type; + c = buffer[start]; + if (c == '\\') + c = next(); + if (c == '/') { + start = start + charlength; + charlength = 1; + if (type == START_COMMENT) { + return COMMENT; + } + return END_COMMENT; + } + } + } + + // Read a number, without checking whether it is out of range + // Doesn't deal with e.g. 0777.9 or 07779f + private int readNumber(char c) { + if (c == '0') { + int saveStart = start, saveLength = charlength; + start = start + charlength; + charlength = 1; + if (start >= end) + return NUMBER; + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + switch (c2) { + case 'x': + case 'X': + start = start + charlength; + charlength = 1; + boolean ok = readDigits(16); + if (!ok) + return bad(NUMBER); + readSuffix(); + return NUMBER; + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + readDigits(8); + readSuffix(); + return NUMBER; + case '.': + case 'e': + case 'E': + start = saveStart; + charlength = saveLength; + break; + case 'f': + case 'F': + case 'd': + case 'D': + start = start + charlength; + charlength = 1; + return NUMBER; + case 'l': + case 'L': + start = start + charlength; + charlength = 1; + return NUMBER; + } + } + boolean hasDigits = false; + if ('0' <= c && c <= '9') { + hasDigits = true; + readDigits(10); + if (start >= end) + return NUMBER; + c = buffer[start]; + if (c == '\\') + c = next(); + if (c == 'l' || c == 'L') { + start = start + charlength; + charlength = 1; + return NUMBER; + } + } + if (c == '.') { + start = start + charlength; + charlength = 1; + if (start >= end) + return NUMBER; + c = buffer[start]; + if (c == '\\') + c = next(); + if ('0' <= c && c <= '9') { + hasDigits = true; + readDigits(10); + if (start >= end) + return NUMBER; + c = buffer[start]; + if (c == '\\') + c = next(); + } + } + if (!hasDigits) + return bad(NUMBER); + switch (c) { + case 'e': + case 'E': + start = start + charlength; + charlength = 1; + if (start >= end) + return bad(NUMBER); + c = buffer[start]; + if (c == '\\') + c = next(); + if (c == '+' || c == '-') { + start = start + charlength; + charlength = 1; + if (start >= end) + return bad(NUMBER); + c = buffer[start]; + if (c == '\\') + c = next(); + } + readDigits(10); + break; + case 'f': + case 'F': + case 'd': + case 'D': + start = start + charlength; + charlength = 1; + return NUMBER; + } + return NUMBER; + } + + boolean readDigits(int radix) { + if (start >= end) + return false; + char c = buffer[start]; + if (c == '\\') + c = next(); + if (Character.digit(c, radix) == -1) + return false; + while (Character.digit(c, radix) != -1) { + start = start + charlength; + charlength = 1; + if (start >= end) + return true; + c = buffer[start]; + if (c == '\\') + c = next(); + } + return true; + } + + void readSuffix() { + if (start >= end) + return; + char c = buffer[start]; + if (c == '\\') + c = next(); + switch (c) { + case 'f': + case 'F': + case 'd': + case 'D': + case 'l': + case 'L': + start = start + charlength; + charlength = 1; + } + } + + private int readDot() { + if (start >= end) + return SEPARATOR; + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (Character.isDigit(c2)) { + return readNumber('.'); + } + if (start + 1 >= end) // || version < 15) + return SEPARATOR; + if (c2 != '.' || buffer[start + 1] != '.') + return SEPARATOR; + start = start + 2; + return SEPARATOR; + } + + private boolean readEscapeSequence() { + if (start >= end) + return false; + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + + switch (c2) { + case 'b': + case 't': + case 'n': + case 'f': + case 'r': + case '\"': + case '\'': + case '\\': + start = start + charlength; + charlength = 1; + return true; + case '0': + case '1': + case '2': + case '3': + return readOctal(3); + case '4': + case '5': + case '6': + case '7': + return readOctal(2); + default: + return false; + } + } + + boolean readOctal(int maxlength) { + if (start >= end) + return false; + char c = buffer[start]; + if (c == '\\') + c = next(); + + int i, val = 0; + for (i = 0; i < maxlength; i++) { + if (Character.digit(c, 8) != -1) { + val = 8 * val + Character.digit(c, 8); + start = start + charlength; + charlength = 1; + if (start >= end) + break; + c = buffer[start]; + if (c == '\\') + c = next(); + } else + break; + } + if ((i == 0) || (val > 0xFF)) + return false; + return true; + } + + // A malformed or incomplete token has a negative type + private int bad(int type) { + return -type; + } + + // Look ahead at the next character or unicode escape. + // For efficiency, replace c = next(); with + // c = buffer[start]; if (c == '\\') c = next(); + // To accept the character after looking at it, use: + // start = start + charlength; charlength = 1; + + // Record the number of source code characters used up. To deal with an odd + // or even number of backslashes preceding a unicode escape, whenever a + // second backslash is coming up, mark its position as a pair. + + private int charlength = 1; + + private int pair = 0; + + private char next() { + if (start >= end) + return 26; // EOF + char c = buffer[start]; + if (c != '\\') + return c; + if (start == pair) { + pair = 0; + return '\\'; + } + if (start + 1 >= end) + return '\\'; + + c = buffer[start + 1]; + if (c == '\\') + pair = start + 1; + if (c != 'u') + return '\\'; + + int pos = start + 2; + while (pos < end && buffer[pos] == 'u') + pos++; + if (pos + 4 > end) { + charlength = end - start; + return '\0'; + } + + c = 0; + for (int j = 0; j < 4; j++) { + int d = Character.digit(buffer[pos + j], 16); + if (d < 0) { + charlength = pos + j - start; + return '\0'; + } + c = (char) (c * 16 + d); + } + charlength = pos + 4 - start; + return c; + } + + // Override initSymbolTable + + protected void initSymbolTable() { + lookup(KEYWORD, "abstract"); + lookup(KEYWORD, "assert"); + lookup(KEYWORD, "boolean"); + lookup(KEYWORD, "break"); + lookup(KEYWORD, "byte"); + lookup(KEYWORD, "case"); + lookup(KEYWORD, "catch"); + lookup(KEYWORD, "char"); + lookup(KEYWORD, "class"); + lookup(KEYWORD, "const"); + lookup(KEYWORD, "continue"); + lookup(KEYWORD, "default"); + lookup(KEYWORD, "do"); + lookup(KEYWORD, "double"); + lookup(KEYWORD, "else"); + lookup(KEYWORD, "enum"); + lookup(KEYWORD, "extends"); + lookup(KEYWORD, "float"); + lookup(KEYWORD, "for"); + lookup(KEYWORD, "goto"); + lookup(KEYWORD, "if"); + lookup(KEYWORD, "int"); + lookup(KEYWORD, "long"); + lookup(KEYWORD, "new"); + lookup(KEYWORD, "private"); + lookup(KEYWORD, "protected"); + lookup(KEYWORD, "public"); + lookup(KEYWORD, "return"); + lookup(KEYWORD, "short"); + lookup(KEYWORD, "static"); + lookup(KEYWORD, "super"); + lookup(KEYWORD, "switch"); + lookup(KEYWORD, "synchronized"); + lookup(KEYWORD, "this"); + lookup(KEYWORD, "throw"); + lookup(KEYWORD, "throws"); + lookup(KEYWORD, "transient"); + lookup(KEYWORD, "try"); + lookup(KEYWORD, "void"); + lookup(KEYWORD, "volatile"); + lookup(KEYWORD, "while"); + + lookup(LITERAL, "TRUE"); + lookup(LITERAL, "FALSE"); + lookup(LITERAL, "NULL"); + lookup(LITERAL, "int8_t"); + lookup(LITERAL, "int16_t"); + lookup(LITERAL, "int32_t"); + lookup(LITERAL, "uint8_t"); + lookup(LITERAL, "uint16_t"); + lookup(LITERAL, "uint32_t"); + lookup(LITERAL, "u8_t"); + lookup(LITERAL, "u16_t"); + lookup(LITERAL, "u32_t"); + } + + // *** Override lookup, but what about unicode escape translation? + + private Symbol temp = new Symbol(0, null); + + protected Symbol lookup(int type, String name) { + if (type != IDENTIFIER) + return super.lookup(type, name); + temp.type = KEYWORD; + temp.name = name; + Symbol sym = symbolTable.get(temp); + if (sym != null) + return sym; + temp.type = LITERAL; + sym = symbolTable.get(temp); + if (sym != null) + return sym; + return super.lookup(type, name); + } + + // Classify the ascii characters using an array of kinds, and classify all + // other unicode characters using an array indexed by unicode category. + // See the source file java/lang/Character.java for the categories. + // To find the classification of a character, use: + // if (c < 128) k = kind[c]; else k = unikind[Character.getType(c)]; + + private static final byte[] kind = new byte[128]; + + private static final byte[] unikind = new byte[31]; + + // Initialise the two classification arrays using static initializer code. + // Token types from the TokenTypes class are used to classify characters. + + private void initKind() { + for (char c = 0; c < 128; c++) + kind[c] = -1; + for (char c = 0; c < 128; c++) + switch (c) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 11: + case 13: + case 14: + case 15: + case 16: + case 17: + case 18: + case 19: + case 20: + case 21: + case 22: + case 23: + case 24: + case 25: + case 27: + case 28: + case 29: + case 30: + case 31: + case 127: + case '#': + case '@': + case '`': + case '\\': + kind[c] = UNRECOGNIZED; + break; + case '\t': + case '\n': + case ' ': + case '\f': + case 26: + kind[c] = WHITESPACE; + break; + case '!': + case '%': + case '&': + case '*': + case '+': + case '-': + case ':': + case '<': + case '=': + case '>': + case '?': + case '^': + case '|': + case '~': + kind[c] = OPERATOR; + break; + case '"': + kind[c] = STRING; + break; + case '\'': + kind[c] = CHARACTER; + break; + case '.': + kind[c] = PUNCTUATION; + break; + case '/': + kind[c] = COMMENT; + break; + case '$': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + kind[c] = IDENTIFIER; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + kind[c] = NUMBER; + break; + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + kind[c] = BRACKET; + break; + case ',': + case ';': + kind[c] = SEPARATOR; + break; + } + for (char c = 0; c < 128; c++) + if (kind[c] == -1) + System.out.println("Char " + ((int) c) + " hasn't been classified"); + } + + private void initUniKind() { + for (byte b = 0; b < 31; b++) + unikind[b] = -1; + for (byte b = 0; b < 31; b++) + switch (b) { + case Character.UNASSIGNED: + case Character.ENCLOSING_MARK: + case Character.OTHER_NUMBER: + case Character.SPACE_SEPARATOR: + case Character.LINE_SEPARATOR: + case Character.PARAGRAPH_SEPARATOR: + case Character.CONTROL: + case 17: // category 17 is unused + case Character.PRIVATE_USE: + case Character.SURROGATE: + case Character.DASH_PUNCTUATION: + case Character.START_PUNCTUATION: + case Character.END_PUNCTUATION: + case Character.OTHER_PUNCTUATION: + case Character.MATH_SYMBOL: + case Character.MODIFIER_SYMBOL: + case Character.OTHER_SYMBOL: + case Character.INITIAL_QUOTE_PUNCTUATION: + case Character.FINAL_QUOTE_PUNCTUATION: + unikind[b] = UNRECOGNIZED; + break; + case Character.UPPERCASE_LETTER: + case Character.LOWERCASE_LETTER: + case Character.TITLECASE_LETTER: + case Character.MODIFIER_LETTER: + case Character.OTHER_LETTER: + case Character.LETTER_NUMBER: + case Character.CONNECTOR_PUNCTUATION: // maybe NUMBER + case Character.CURRENCY_SYMBOL: + // Characters where Other_ID_Start is true + unikind[b] = IDENTIFIER; + break; + case Character.NON_SPACING_MARK: + case Character.COMBINING_SPACING_MARK: + case Character.DECIMAL_DIGIT_NUMBER: + case Character.FORMAT: + unikind[b] = NUMBER; + break; + } + for (byte b = 0; b < 31; b++) + if (unikind[b] == -1) + System.out.println("Unicode cat " + b + " hasn't been classified"); + } + +} diff --git a/se/sics/mspsim/extutil/highlight/HighlightSourceViewer.java b/se/sics/mspsim/extutil/highlight/HighlightSourceViewer.java new file mode 100644 index 0000000..85fd191 --- /dev/null +++ b/se/sics/mspsim/extutil/highlight/HighlightSourceViewer.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2007, Swedish Institute of Computer Science. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: $ + * + * ----------------------------------------------------------------- + * + * HighlightSourceViewer + * + * Authors : Adam Dunkels, Joakim Eriksson, Niclas Finne + * Created : 6 dec 2007 + * Updated : $Date: 6 dec 2007 $ + * $Revision: 1.0 $ + */ + +package se.sics.mspsim.extutil.highlight; + +import java.awt.Container; +import java.io.FileReader; +import java.io.IOException; + +import javax.swing.JFrame; +import javax.swing.JOptionPane; +import javax.swing.JScrollPane; +import javax.swing.SwingUtilities; + +import se.sics.mspsim.util.SourceViewer; + +/** + * + */ +public class HighlightSourceViewer implements SourceViewer { + + private JFrame window; + private SyntaxHighlighter highlighter; + + public HighlightSourceViewer() { + // + } + + private void setup() { + if (window == null) { + window = new JFrame("Source Viewer"); + window.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); + + Scanner scanner = new CScanner(); + highlighter = new SyntaxHighlighter(24, 80, scanner); + highlighter.setBorder(new LineNumberedBorder(LineNumberedBorder.LEFT_SIDE, LineNumberedBorder.RIGHT_JUSTIFY)); + JScrollPane scroller = new JScrollPane(highlighter); + scroller.setVerticalScrollBarPolicy(JScrollPane.VERTICAL_SCROLLBAR_ALWAYS); + Container pane = window.getContentPane(); + pane.add(scroller); + window.pack(); + } + } + + public boolean isVisible() { + return window != null && window.isVisible(); + } + + public void setVisible(boolean isVisible) { + setup(); + window.setVisible(isVisible); + } + + public void viewFile(final String file) { + setup(); + SwingUtilities.invokeLater(new Runnable() { + public void run() { + try { + FileReader reader = new FileReader(file); + try { + highlighter.read(reader, null); + // Workaround for bug 4782232 in Java 1.4 + highlighter.setCaretPosition(1); + highlighter.setCaretPosition(0); + } finally { + reader.close(); + } + } catch (IOException err) { + err.printStackTrace(); + JOptionPane.showMessageDialog(window, "Failed to read the file '" + file + '\'', "Could not read file", JOptionPane.ERROR_MESSAGE); + } + } + }); + } + + public void viewLine(final int line) { + if (highlighter != null) { + SwingUtilities.invokeLater(new Runnable() { + public void run() { + if (line >= 0 && line < highlighter.getLineCount()) { + highlighter.setCaretPosition(highlighter.getLineStartOffset(line)); + } + } + }); + } + } + + public static void main(String[] args) { + HighlightSourceViewer sv = new HighlightSourceViewer(); + sv.setVisible(true); + sv.viewFile(args[0]); + } +} diff --git a/se/sics/mspsim/extutil/highlight/JavaScanner.java b/se/sics/mspsim/extutil/highlight/JavaScanner.java new file mode 100644 index 0000000..139b14b --- /dev/null +++ b/se/sics/mspsim/extutil/highlight/JavaScanner.java @@ -0,0 +1,962 @@ +package se.sics.mspsim.extutil.highlight; +// Public domain, no restrictions, Ian Holyer, University of Bristol. + +/** + *
+ * Provide a hand-written scanner for the Java language. + */ + +public class JavaScanner extends Scanner { + + // The version of Java supported. + private int version = 15; + + private boolean debug = false; + + /** Create a Java scanner, for Java version 1.5 by default. */ + public JavaScanner() { + super(); + initKind(); + initUniKind(); + } + + /** Create a Java scanner, for a given version between "1.1" and "1.5". */ + public JavaScanner(String version) { + super(); + initKind(); + initUniKind(); + if (version.equals("1.1")) + this.version = 11; + else if (version.equals("1.2")) + this.version = 12; + else if (version.equals("1.3")) + this.version = 13; + else if (version.equals("1.4")) + this.version = 14; + else if (version.equals("1.5")) + this.version = 15; + else + throw new Error("Unknown version of Java: " + version); + } + + /** Override the read method from the Scanner class. */ + protected int read() { + int type, saveStart = 0; + if (debug) + saveStart = start; + + if (start >= end) + return WHITESPACE; + + switch (state) { + case MID_COMMENT: + case END_COMMENT: + type = readComment(MID_COMMENT); + if (type == END_COMMENT) + state = WHITESPACE; + else + state = MID_COMMENT; + return type; + default: + char c = buffer[start]; + if (c == '\\') + c = next(); + if (c < 128) + type = kind[c]; + else + type = unikind[Character.getType(c)]; + switch (type) { + case WHITESPACE: + start = start + charlength; + charlength = 1; + while (start < end) { + c = buffer[start]; + if (c == '\\') + c = next(); + int k; + if (c < 128) + k = kind[c]; + else + k = unikind[Character.getType(c)]; + if (k != WHITESPACE) + break; + start = start + charlength; + charlength = 1; + } + break; + case UNRECOGNIZED: + case BRACKET: + case SEPARATOR: + start = start + charlength; + charlength = 1; + break; + case OPERATOR: + start = start + charlength; + charlength = 1; + type = readOperator(c); + break; + case CHARACTER: + start = start + charlength; + charlength = 1; + type = readCharLiteral(); + break; + case STRING: + start = start + charlength; + charlength = 1; + type = readStringLiteral(); + break; + case IDENTIFIER: + start = start + charlength; + charlength = 1; + while (start < end) { + c = buffer[start]; + if (c == '\\') + c = next(); + int k; + if (c < 128) + k = kind[c]; + else + k = unikind[Character.getType(c)]; + if (k != IDENTIFIER && k != NUMBER) + break; + start = start + charlength; + charlength = 1; + } + break; + case NUMBER: + start = start + charlength; + charlength = 1; + type = readNumber(c); + break; + case PUNCTUATION: + start = start + charlength; + charlength = 1; + type = readDot(); + break; + case COMMENT: + start = start + charlength; + charlength = 1; + type = readSlash(); + if (type == START_COMMENT) + state = MID_COMMENT; + break; + } + } + if (debug) { + System.out.print(TokenTypes.typeNames[type]); + System.out.println(" " + saveStart + "," + end + "(" + + (start - saveStart) + ")"); + } + return type; + } + + private int readOperator(char c) { + if (start >= end) + return OPERATOR; + char c2; + + switch (c) { + case '~': + case '?': + case ':': + break; + case '+': + case '-': + case '&': + case '|': + c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (c2 != c && c2 != '=') + break; + start = start + charlength; + charlength = 1; + break; + case '=': + case '*': + case '!': + case '^': + case '%': + case '/': + c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (c2 != '=') + break; + start = start + charlength; + charlength = 1; + break; + case '<': + case '>': + c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (c2 == '=') { + start = start + charlength; + charlength = 1; + } else if (c2 == c) { + start = start + charlength; + charlength = 1; + if (start >= end) + break; + char c3 = buffer[start]; + if (c3 == '\\') + c3 = next(); + if (c3 == '=') { + start = start + charlength; + charlength = 1; + } else if (c == '>' && c3 == '>') // >>> + { + start = start + charlength; + charlength = 1; + if (start >= end) + break; + char c4 = buffer[start]; + if (c4 == '\\') + c4 = next(); + if (c4 != '=') + break; + start = start + charlength; + charlength = 1; + } + } + break; + } + return OPERATOR; + } + + private int readCharLiteral() { + if (start >= end) + return bad(CHARACTER); + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + + switch (c2) { + case '\\': + start = start + charlength; + charlength = 1; + boolean ok = readEscapeSequence(); + if (!ok) + return bad(CHARACTER); + break; + case '\'': + case '\n': + return bad(CHARACTER); + default: + start = start + charlength; + charlength = 1; + break; + } + if (start >= end) + return bad(CHARACTER); + char c3 = buffer[start]; + if (c3 == '\\') + c3 = next(); + if (c3 != '\'') + return bad(CHARACTER); + start = start + charlength; + charlength = 1; + return CHARACTER; + } + + private int readStringLiteral() { + if (start >= end) + return bad(STRING); + char c = buffer[start]; + if (c == '\\') + c = next(); + + while (c != '"') { + switch (c) { + case '\\': + start = start + charlength; + charlength = 1; + boolean ok = readEscapeSequence(); + if (!ok) + return bad(STRING); + break; + case '\n': + return bad(STRING); + default: + start = start + charlength; + charlength = 1; + if (start >= end) + return bad(STRING); + break; + } + c = buffer[start]; + if (c == '\\') + c = next(); + } + if (c != '"') + return bad(STRING); + start = start + charlength; + charlength = 1; + return STRING; + } + + private int readSlash() { + if (start >= end) + return OPERATOR; + char c = buffer[start]; + if (c == '\\') + c = next(); + if (c == '/') { + while (c != '\n') { + start = start + charlength; + charlength = 1; + if (start >= end) + return COMMENT; + c = buffer[start]; + if (c == '\\') + c = next(); + } + start = start + charlength; + charlength = 1; + return COMMENT; + } else if (c == '*') { + start = start + charlength; + charlength = 1; + return readComment(START_COMMENT); + } + return readOperator('/'); + } + + // Read one line of a /*...*/ comment, given the expected type + int readComment(int type) { + if (start >= end) + return type; + char c = buffer[start]; + if (c == '\\') + c = next(); + + while (true) { + while (c != '*' && c != '\n') { + start = start + charlength; + charlength = 1; + if (start >= end) + return type; + c = buffer[start]; + if (c == '\\') + c = next(); + } + start = start + charlength; + charlength = 1; + if (c == '\n') + return type; + if (start >= end) + return type; + c = buffer[start]; + if (c == '\\') + c = next(); + if (c == '/') { + start = start + charlength; + charlength = 1; + if (type == START_COMMENT) { + return COMMENT; + } + return END_COMMENT; + } + } + } + + // Read a number, without checking whether it is out of range + // Doesn't deal with e.g. 0777.9 or 07779f + private int readNumber(char c) { + if (c == '0') { + int saveStart = start, saveLength = charlength; + start = start + charlength; + charlength = 1; + if (start >= end) + return NUMBER; + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + switch (c2) { + case 'x': + case 'X': + start = start + charlength; + charlength = 1; + boolean ok = readDigits(16); + if (!ok) + return bad(NUMBER); + readSuffix(); + return NUMBER; + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + readDigits(8); + readSuffix(); + return NUMBER; + case '.': + case 'e': + case 'E': + start = saveStart; + charlength = saveLength; + break; + case 'f': + case 'F': + case 'd': + case 'D': + start = start + charlength; + charlength = 1; + return NUMBER; + case 'l': + case 'L': + start = start + charlength; + charlength = 1; + return NUMBER; + } + } + boolean hasDigits = false; + if ('0' <= c && c <= '9') { + hasDigits = true; + readDigits(10); + if (start >= end) + return NUMBER; + c = buffer[start]; + if (c == '\\') + c = next(); + if (c == 'l' || c == 'L') { + start = start + charlength; + charlength = 1; + return NUMBER; + } + } + if (c == '.') { + start = start + charlength; + charlength = 1; + if (start >= end) + return NUMBER; + c = buffer[start]; + if (c == '\\') + c = next(); + if ('0' <= c && c <= '9') { + hasDigits = true; + readDigits(10); + if (start >= end) + return NUMBER; + c = buffer[start]; + if (c == '\\') + c = next(); + } + } + if (!hasDigits) + return bad(NUMBER); + switch (c) { + case 'e': + case 'E': + start = start + charlength; + charlength = 1; + if (start >= end) + return bad(NUMBER); + c = buffer[start]; + if (c == '\\') + c = next(); + if (c == '+' || c == '-') { + start = start + charlength; + charlength = 1; + if (start >= end) + return bad(NUMBER); + c = buffer[start]; + if (c == '\\') + c = next(); + } + readDigits(10); + break; + case 'f': + case 'F': + case 'd': + case 'D': + start = start + charlength; + charlength = 1; + return NUMBER; + } + return NUMBER; + } + + boolean readDigits(int radix) { + if (start >= end) + return false; + char c = buffer[start]; + if (c == '\\') + c = next(); + if (Character.digit(c, radix) == -1) + return false; + while (Character.digit(c, radix) != -1) { + start = start + charlength; + charlength = 1; + if (start >= end) + return true; + c = buffer[start]; + if (c == '\\') + c = next(); + } + return true; + } + + void readSuffix() { + if (start >= end) + return; + char c = buffer[start]; + if (c == '\\') + c = next(); + switch (c) { + case 'f': + case 'F': + case 'd': + case 'D': + case 'l': + case 'L': + start = start + charlength; + charlength = 1; + } + } + + private int readDot() { + if (start >= end) + return SEPARATOR; + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + if (Character.isDigit(c2)) { + return readNumber('.'); + } + if (start + 1 >= end || version < 15) + return SEPARATOR; + if (c2 != '.' || buffer[start + 1] != '.') + return SEPARATOR; + start = start + 2; + return SEPARATOR; + } + + private boolean readEscapeSequence() { + if (start >= end) + return false; + char c2 = buffer[start]; + if (c2 == '\\') + c2 = next(); + + switch (c2) { + case 'b': + case 't': + case 'n': + case 'f': + case 'r': + case '\"': + case '\'': + case '\\': + start = start + charlength; + charlength = 1; + return true; + case '0': + case '1': + case '2': + case '3': + return readOctal(3); + case '4': + case '5': + case '6': + case '7': + return readOctal(2); + default: + return false; + } + } + + boolean readOctal(int maxlength) { + if (start >= end) + return false; + char c = buffer[start]; + if (c == '\\') + c = next(); + + int i, val = 0; + for (i = 0; i < maxlength; i++) { + if (Character.digit(c, 8) != -1) { + val = 8 * val + Character.digit(c, 8); + start = start + charlength; + charlength = 1; + if (start >= end) + break; + c = buffer[start]; + if (c == '\\') + c = next(); + } else + break; + } + if ((i == 0) || (val > 0xFF)) + return false; + return true; + } + + // A malformed or incomplete token has a negative type + private int bad(int type) { + return -type; + } + + // Look ahead at the next character or unicode escape. + // For efficiency, replace c = next(); with + // c = buffer[start]; if (c == '\\') c = next(); + // To accept the character after looking at it, use: + // start = start + charlength; charlength = 1; + + // Record the number of source code characters used up. To deal with an odd + // or even number of backslashes preceding a unicode escape, whenever a + // second backslash is coming up, mark its position as a pair. + + private int charlength = 1; + + private int pair = 0; + + private char next() { + if (start >= end) + return 26; // EOF + char c = buffer[start]; + if (c != '\\') + return c; + if (start == pair) { + pair = 0; + return '\\'; + } + if (start + 1 >= end) + return '\\'; + + c = buffer[start + 1]; + if (c == '\\') + pair = start + 1; + if (c != 'u') + return '\\'; + + int pos = start + 2; + while (pos < end && buffer[pos] == 'u') + pos++; + if (pos + 4 > end) { + charlength = end - start; + return '\0'; + } + + c = 0; + for (int j = 0; j < 4; j++) { + int d = Character.digit(buffer[pos + j], 16); + if (d < 0) { + charlength = pos + j - start; + return '\0'; + } + c = (char) (c * 16 + d); + } + charlength = pos + 4 - start; + return c; + } + + // Override initSymbolTable + + protected void initSymbolTable() { + lookup(KEYWORD, "abstract"); + if (version >= 14) + lookup(KEYWORD, "assert"); + lookup(KEYWORD, "boolean"); + lookup(KEYWORD, "break"); + lookup(KEYWORD, "byte"); + lookup(KEYWORD, "case"); + lookup(KEYWORD, "catch"); + lookup(KEYWORD, "char"); + lookup(KEYWORD, "class"); + lookup(KEYWORD, "const"); + lookup(KEYWORD, "continue"); + lookup(KEYWORD, "default"); + lookup(KEYWORD, "do"); + lookup(KEYWORD, "double"); + lookup(KEYWORD, "else"); + if (version >= 15) + lookup(KEYWORD, "enum"); + lookup(KEYWORD, "extends"); + lookup(KEYWORD, "final"); + lookup(KEYWORD, "finally"); + lookup(KEYWORD, "float"); + lookup(KEYWORD, "for"); + lookup(KEYWORD, "goto"); + lookup(KEYWORD, "if"); + lookup(KEYWORD, "implements"); + lookup(KEYWORD, "import"); + lookup(KEYWORD, "instanceof"); + lookup(KEYWORD, "int"); + lookup(KEYWORD, "interface"); + lookup(KEYWORD, "long"); + lookup(KEYWORD, "native"); + lookup(KEYWORD, "new"); + lookup(KEYWORD, "package"); + lookup(KEYWORD, "private"); + lookup(KEYWORD, "protected"); + lookup(KEYWORD, "public"); + lookup(KEYWORD, "return"); + lookup(KEYWORD, "short"); + lookup(KEYWORD, "static"); + if (version >= 12) + lookup(KEYWORD, "strictfp"); + lookup(KEYWORD, "super"); + lookup(KEYWORD, "switch"); + lookup(KEYWORD, "synchronized"); + lookup(KEYWORD, "this"); + lookup(KEYWORD, "throw"); + lookup(KEYWORD, "throws"); + lookup(KEYWORD, "transient"); + lookup(KEYWORD, "try"); + lookup(KEYWORD, "void"); + lookup(KEYWORD, "volatile"); + lookup(KEYWORD, "while"); + + lookup(LITERAL, "true"); + lookup(LITERAL, "false"); + lookup(LITERAL, "null"); + } + + // *** Override lookup, but what about unicode escape translation? + + private Symbol temp = new Symbol(0, null); + + protected Symbol lookup(int type, String name) { + if (type != IDENTIFIER) + return super.lookup(type, name); + temp.type = KEYWORD; + temp.name = name; + Symbol sym = symbolTable.get(temp); + if (sym != null) + return sym; + temp.type = LITERAL; + sym = symbolTable.get(temp); + if (sym != null) + return sym; + return super.lookup(type, name); + } + + // Classify the ascii characters using an array of kinds, and classify all + // other unicode characters using an array indexed by unicode category. + // See the source file java/lang/Character.java for the categories. + // To find the classification of a character, use: + // if (c < 128) k = kind[c]; else k = unikind[Character.getType(c)]; + + private static final byte[] kind = new byte[128]; + + private static final byte[] unikind = new byte[31]; + + // Initialise the two classification arrays using static initializer code. + // Token types from the TokenTypes class are used to classify characters. + + private void initKind() { + for (char c = 0; c < 128; c++) + kind[c] = -1; + for (char c = 0; c < 128; c++) + switch (c) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 11: + case 13: + case 14: + case 15: + case 16: + case 17: + case 18: + case 19: + case 20: + case 21: + case 22: + case 23: + case 24: + case 25: + case 27: + case 28: + case 29: + case 30: + case 31: + case 127: + case '#': + case '@': + case '`': + case '\\': + kind[c] = UNRECOGNIZED; + break; + case '\t': + case '\n': + case ' ': + case '\f': + case 26: + kind[c] = WHITESPACE; + break; + case '!': + case '%': + case '&': + case '*': + case '+': + case '-': + case ':': + case '<': + case '=': + case '>': + case '?': + case '^': + case '|': + case '~': + kind[c] = OPERATOR; + break; + case '"': + kind[c] = STRING; + break; + case '\'': + kind[c] = CHARACTER; + break; + case '.': + kind[c] = PUNCTUATION; + break; + case '/': + kind[c] = COMMENT; + break; + case '$': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + kind[c] = IDENTIFIER; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + kind[c] = NUMBER; + break; + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + kind[c] = BRACKET; + break; + case ',': + case ';': + kind[c] = SEPARATOR; + break; + } + for (char c = 0; c < 128; c++) + if (kind[c] == -1) + System.out.println("Char " + ((int) c) + " hasn't been classified"); + } + + private void initUniKind() { + for (byte b = 0; b < 31; b++) + unikind[b] = -1; + for (byte b = 0; b < 31; b++) + switch (b) { + case Character.UNASSIGNED: + case Character.ENCLOSING_MARK: + case Character.OTHER_NUMBER: + case Character.SPACE_SEPARATOR: + case Character.LINE_SEPARATOR: + case Character.PARAGRAPH_SEPARATOR: + case Character.CONTROL: + case 17: // category 17 is unused + case Character.PRIVATE_USE: + case Character.SURROGATE: + case Character.DASH_PUNCTUATION: + case Character.START_PUNCTUATION: + case Character.END_PUNCTUATION: + case Character.OTHER_PUNCTUATION: + case Character.MATH_SYMBOL: + case Character.MODIFIER_SYMBOL: + case Character.OTHER_SYMBOL: + case Character.INITIAL_QUOTE_PUNCTUATION: + case Character.FINAL_QUOTE_PUNCTUATION: + unikind[b] = UNRECOGNIZED; + break; + case Character.UPPERCASE_LETTER: + case Character.LOWERCASE_LETTER: + case Character.TITLECASE_LETTER: + case Character.MODIFIER_LETTER: + case Character.OTHER_LETTER: + case Character.LETTER_NUMBER: + case Character.CONNECTOR_PUNCTUATION: // maybe NUMBER + case Character.CURRENCY_SYMBOL: + // Characters where Other_ID_Start is true + unikind[b] = IDENTIFIER; + break; + case Character.NON_SPACING_MARK: + case Character.COMBINING_SPACING_MARK: + case Character.DECIMAL_DIGIT_NUMBER: + case Character.FORMAT: + unikind[b] = NUMBER; + break; + } + for (byte b = 0; b < 31; b++) + if (unikind[b] == -1) + System.out.println("Unicode cat " + b + " hasn't been classified"); + } + +} diff --git a/se/sics/mspsim/extutil/highlight/LineNumberedBorder.java b/se/sics/mspsim/extutil/highlight/LineNumberedBorder.java new file mode 100644 index 0000000..c5624ef --- /dev/null +++ b/se/sics/mspsim/extutil/highlight/LineNumberedBorder.java @@ -0,0 +1,258 @@ +package se.sics.mspsim.extutil.highlight; +import java.awt.Component; +import java.awt.FontMetrics; +import java.awt.Graphics; +import java.awt.Insets; + +import javax.swing.border.AbstractBorder; + +/** + * Draws line numbers next to each line, in the same font as the text. + * Currently, this can only be used with a SyntaxHighlighter , since it + * relies on the getRows() and getLineCount() methods. A + * possible extension, create an interface to return this rows/linecount. + * + * @author Paul Durbin (McDurby@yahoo.com) + * @created January 29, 2002 + */ +public class LineNumberedBorder extends AbstractBorder { + + private static final long serialVersionUID = -3812536735962506061L; + + /** + * The line numbers should be drawn on the left side of the component. + */ + public static int LEFT_SIDE = -2; + + /** + * The line numbers should be drawn on the right side of the component. + */ + public static int RIGHT_SIDE = -1; + + /** + * The line number should be right justified. + */ + public static int RIGHT_JUSTIFY = 0; + + /** + * The line number should be left justified. + */ + public static int LEFT_JUSTIFY = 1; + + /** + * Indicates the justification of the text of the line number. + */ + private int lineNumberJustification = RIGHT_JUSTIFY; + + /** + * Indicates the location of the line numbers, w.r.t. the component. + */ + private int location = LEFT_SIDE; + + public LineNumberedBorder(int location, int justify) { + setLocation(location); + setLineNumberJustification(justify); + } + + public Insets getBorderInsets(Component c) { + return getBorderInsets(c, new Insets(0, 0, 0, 0)); + } + + /** + * This modifies the insets, by adding space for the line number on the left. + * Should be modified to add space on the right, depending upon Locale. + * + * @param c + * Description of the Parameter + * @param insets + * Description of the Parameter + * @return The borderInsets value + */ + public Insets getBorderInsets(Component c, Insets insets) { + // if c is not a SyntaxHighlighter...nothing is done... + if (c instanceof SyntaxHighlighter) { + int width = lineNumberWidth((SyntaxHighlighter) c); + if (location == LEFT_SIDE) { + insets.left = width; + } else { + insets.right = width; + } + } + return insets; + } + + public int getLineNumberJustification() { + return lineNumberJustification; + } + + public void setLineNumberJustification(int justify) { + if (justify == RIGHT_JUSTIFY || justify == LEFT_JUSTIFY) { + lineNumberJustification = justify; + } + } + + public int getLocation() { + return location; + } + + public void setLocation(int loc) { + if (loc == RIGHT_SIDE || loc == LEFT_SIDE) { + location = loc; + } + } + + /** + * Returns the width, in pixels, of the maximum line number, plus a trailing + * space. + * + * @param textArea + * Description of the Parameter + * @return Description of the Return Value + */ + private int lineNumberWidth(SyntaxHighlighter textArea) { + // + // note: should this be changed to use all nines for the lineCount? + // for example, if the number of rows is 111...999 could be wider + // (in pixels) in a proportionally spaced font... + // + int lineCount = Math.max(textArea.getRows(), textArea.getLineCount() + 1); + return textArea.getFontMetrics(textArea.getFont()).stringWidth( + lineCount + " "); + } + + // + // NOTE: This method is called every time the cursor blinks... + // so...optimize (later and if possible) for speed... + // + public void paintBorder(Component c, Graphics g, int x, int y, int width, + int height) { + + java.awt.Rectangle clip = g.getClipBounds(); + + FontMetrics fm = g.getFontMetrics(); + int fontHeight = fm.getHeight(); + + // starting location at the "top" of the page... + // y is the starting baseline for the font... + // should "font leading" be applied? + int ybaseline = y + fm.getAscent(); + + // + // now determine if it is the "top" of the page...or somewhere else + // + int startingLineNumber = (clip.y / fontHeight) + 1; + + // + // use any one of the following if's: + // + // if (startingLineNumber != 1) + if (ybaseline < clip.y) { + // + // not within the clip rectangle...move it... + // determine how many fontHeight's there are between + // y and clip.y...then add that many fontHeights + // + ybaseline = + y + startingLineNumber * fontHeight - (fontHeight - fm.getAscent()); + } + + // + // options: + // . write the number rows in the document (current) + // . write the number of existing lines in the document (to do) + // see getLineCount() + // + + // determine which the "drawing" should end... + // add fontHeight: make sure...part of the line number is drawn + // + // could also do this by determining what the last line + // number to draw. + // then the "while" loop whould change accordingly. + // + // int yend = y + clip.height + fontHeight; + // int yend = ybaseline + height + fontHeight; // original + int yend = ybaseline + height; + if (yend > (y + height)) { + yend = y + height; + } + + SyntaxHighlighter jta = (SyntaxHighlighter) c; + int lineWidth = lineNumberWidth(jta); + + // base x position of the line number + int lnxstart = x; + if (location == LEFT_SIDE) { + // x (LEFT) or (x + lineWidth) (RIGHT) + // (depends upon justification) + if (lineNumberJustification == LEFT_JUSTIFY) { + lnxstart = x; + } else { + // RIGHT JUSTIFY + lnxstart = x + lineWidth; + } + } else { + // RIGHT SIDE + // (y + width) - lineWidth (LEFT) or (y + width) (RIGHT) + // (depends upon justification) + if (lineNumberJustification == LEFT_JUSTIFY) { + lnxstart = (y + width) - lineWidth; + } else { + // RIGHT JUSTIFY + lnxstart = (y + width); + } + } + + g.setColor(c.getForeground()); + // + // loop until out of the "visible" region... + // + int length = + ("" + Math.max(jta.getRows(), jta.getLineCount() + 1)).length(); + while (ybaseline < yend) { + // + // options: + // . left justify the line numbers + // . right justify the line numbers + // + + if (lineNumberJustification == LEFT_JUSTIFY) { + g.drawString(startingLineNumber + " ", lnxstart, ybaseline); + } else { + // right justify + String label = padLabel(startingLineNumber, length, true); + g.drawString(label, lnxstart - fm.stringWidth(label), ybaseline); + } + + ybaseline += fontHeight; + startingLineNumber++; + } + } + + // paintComponent + + /** + * Create the string for the line number. NOTE: The length param + * does not include the optional space added after the line number. + * + * @param lineNumber + * to stringize + * @param length + * the length desired of the string + * @param addSpace + * Description of the Parameter + * @return the line number for drawing + */ + private static String padLabel(int lineNumber, int length, boolean addSpace) { + StringBuffer buffer = new StringBuffer(); + buffer.append(lineNumber); + for (int count = (length - buffer.length()); count > 0; count--) { + buffer.insert(0, ' '); + } + if (addSpace) { + buffer.append(' '); + } + return buffer.toString(); + } +} +// LineNumberedBorder diff --git a/se/sics/mspsim/extutil/highlight/Scan.java b/se/sics/mspsim/extutil/highlight/Scan.java new file mode 100644 index 0000000..8d54c90 --- /dev/null +++ b/se/sics/mspsim/extutil/highlight/Scan.java @@ -0,0 +1,38 @@ +package se.sics.mspsim.extutil.highlight; +// Illustrate the use of the scanner by reading in a file and displaying its +// tokens. Public domain, no restrictions, Ian Holyer, University of Bristol. + +import java.io.*; + +public class Scan { + // Get the filename from the command line + public static void main(String[] args) throws IOException { + Scan program = new Scan(); + if (args.length != 1) { + System.out.println("Usage: java Scan filename"); + } else { + program.scan(args[0]); + } + } + + // Scan each line in turn + public void scan(String filename) throws IOException { + File file = new File(filename); + int len = (int) file.length(); + char[] buffer = new char[len]; + Reader in = new FileReader(file); + in.read(buffer); + in.close(); + + Scanner scanner = new Scanner(); + scanner.change(0, 0, len); + scanner.scan(buffer, 0, len); + + for (int i = 0; i < scanner.size(); i++) { + Token t = scanner.getToken(i); + System.out.print("" + t.position); + System.out.print(": " + t.symbol.name); + System.out.println(" " + TokenTypes.typeNames[t.symbol.type]); + } + } +} diff --git a/se/sics/mspsim/extutil/highlight/Scanner.java b/se/sics/mspsim/extutil/highlight/Scanner.java new file mode 100644 index 0000000..dc370ca --- /dev/null +++ b/se/sics/mspsim/extutil/highlight/Scanner.java @@ -0,0 +1,450 @@ +package se.sics.mspsim.extutil.highlight; +import java.util.HashMap; + +// Public domain, no restrictions, Ian Holyer, University of Bristol. + +/** + *
+ * A Scanner object provides a lexical analyser and a resulting token array. + * Incremental rescanning is supported, e.g. for use in a token colouring + * editor. This is a base class dealing with plain text, which can be extended + * to support other languages. + * + *
+ * The actual text is assumed to be held elsewhere, e.g. in a document. The
+ * change() method is called to report the position and length of
+ * a change in the text, and the scan() method is called to
+ * perform scanning or rescanning. For example, to scan an entire document held
+ * in a character array text in one go:
+ *
+ *
+ * + *+ * + *+ * scanner.change(0, 0, text.length); + * scanner.scan(text, 0, text.length); + *+ * + *
+ * For incremental scanning, the position() method is used to
+ * find the text position at which rescanning should start. For example, a
+ * syntax highlighter might contain this code:
+ *
+ *
+ *
+ *
+ * // Where to start rehighlighting, and a segment object
+ * int firstRehighlightToken;
+ * Segment segment;
+ * ...
+ * // Whenever the text changes, e.g. on an insert or remove or read.
+ * firstRehighlightToken = scanner.change(offset, oldLength, newLength);
+ * repaint();
+ * ...
+ * // in repaintComponent
+ * int offset = scanner.position();
+ * if (offset < 0) return;
+ * int tokensToRedo = 0;
+ * int amount = 100;
+ * while (tokensToRedo == 0 && offset >= 0)
+ * {
+ * int length = doc.getLength() - offset;
+ * if (length > amount) length = amount;
+ * try { doc.getText(offset, length, text); }
+ * catch (BadLocationException e) { return; }
+ * tokensToRedo = scanner.scan(text.array, text.offset, text.count);
+ * offset = scanner.position();
+ * amount = 2*amount;
+ * }
+ * for (int i = 0; i < tokensToRedo; i++)
+ * {
+ * Token t = scanner.getToken(firstRehighlightToken + i);
+ * int length = t.symbol.name.length();
+ * int type = t.symbol.type;
+ * doc.setCharacterAttributes (t.position, length, styles[type], false);
+ * }
+ * firstRehighlightToken += tokensToRedo;
+ * if (offset >= 0) repaint(2);
+ *
+ *
+ *
+ *
+ *
+ * Note that change can be called at any time, even between calls
+ * to scan. Only small number of characters are passed to
+ * scan so that only a small burst of scanning is done, to
+ * prevent the program's user interface from freezing.
+ */
+public class Scanner implements TokenTypes {
+ /**
+ *
+ * Read one token from the start of the current text buffer, given the start + * offset, end offset, and current scanner state. The method moves the start + * offset past the token, updates the scanner state, and returns the type of + * the token just scanned. + * + *
+ * The scanner state is a representative token type. It is either the state
+ * left after the last call to read, or the type of the old token at the same
+ * position if rescanning, or WHITESPACE if at the start of a document. The
+ * method succeeds in all cases, returning whitespace or comment or error
+ * tokens where necessary. Each line of a multi-line comment is treated as a
+ * separate token, to improve incremental rescanning. If the buffer does not
+ * extend to the end of the document, the last token returned for the buffer
+ * may be incomplete and the caller must rescan it. The read method can be
+ * overridden to implement different languages. The default version splits
+ * plain text into words, numbers and punctuation.
+ */
+ protected int read() {
+ char c = buffer[start];
+ int type;
+ // Ignore the state, since there is only one.
+ if (Character.isWhitespace(c)) {
+ type = WHITESPACE;
+ while (++start < end) {
+ if (!Character.isWhitespace(buffer[start]))
+ break;
+ }
+ } else if (Character.isLetter(c)) {
+ type = WORD;
+ while (++start < end) {
+ c = buffer[start];
+ if (Character.isLetter(c) || Character.isDigit(c))
+ continue;
+ if (c == '-' || c == '\'' || c == '_')
+ continue;
+ break;
+ }
+ } else if (Character.isDigit(c)) {
+ type = NUMBER;
+ while (++start < end) {
+ c = buffer[start];
+ if (!Character.isDigit(c) && c != '.')
+ break;
+ }
+ } else if (c >= '!' || c <= '~') {
+ type = PUNCTUATION;
+ start++;
+ } else {
+ type = UNRECOGNIZED;
+ start++;
+ }
+
+ // state = WHITESPACE;
+ return type;
+ }
+
+ /**
+ * The current buffer of text being scanned.
+ */
+ protected char[] buffer;
+
+ /**
+ * The current offset within the buffer, at which to scan the next token.
+ */
+ protected int start;
+
+ /**
+ * The end offset in the buffer.
+ */
+ protected int end;
+
+ /**
+ * The current scanner state, as a representative token type.
+ */
+ protected int state = WHITESPACE;
+
+ // The array of tokens forms a gap buffer. The total length of the text is
+ // tracked, and tokens after the gap have (negative) positions relative to
+ // the end of the text. While scanning, the gap represents the area to be
+ // scanned, no tokens after the gap can be taken as valid, and in particular
+ // the end-of-text sentinel token is after the gap.
+
+ private Token[] tokens;
+
+ private int gap, endgap, textLength;
+
+ private boolean scanning;
+
+ private int position;
+
+ /**
+ * The symbol table can be accessed by
+ * The UNRECOGNIZED constant (zero) is for tokens which are completely
+ * unrecognized, usually consisting of a single illegal character. Other error
+ * tokens are represented by negative types, where -t represents an incomplete
+ * or malformed token of type t. An error token usually consists of the maximal
+ * legal substring of the source text.
+ *
+ *
+ * The WHITESPACE constant is used to classify tokens which are to be discarded,
+ * it acts as a suitable scanner state at the beginning of a document, and it is
+ * used for the usual end-of-text sentinel token which marks the end of the
+ * document. Comments can optionally be classified as WHITESPACE and discarded,
+ * if they are not needed for highlighting. No other types besides UNRECOGNIZED
+ * and WHITESPACE are treated specially.
+ *
+ *
+ * The constants are presented as an interface so that any class can implement
+ * the interface and use the names of the constants directly, without prefixing
+ * them with a class name.
+ *
+ */
+public interface TokenTypes {
+ public static final int UNRECOGNIZED = 0, WHITESPACE = 1, WORD = 2,
+ NUMBER = 3, PUNCTUATION = 4, COMMENT = 5, START_COMMENT = 6,
+ MID_COMMENT = 7, END_COMMENT = 8, TAG = 9, END_TAG = 10, KEYWORD = 11,
+ KEYWORD2 = 12, IDENTIFIER = 13, LITERAL = 14, STRING = 15,
+ CHARACTER = 16, OPERATOR = 17, BRACKET = 18, SEPARATOR = 19, URL = 20;
+
+ /**
+ * The names of the token types, indexed by type, are provided for descriptive
+ * purposes.
+ */
+ public static final String[] typeNames = {
+ "bad token", "whitespace", "word", "number", "punctuation", "comment",
+ "start of comment", "middle of comment", "end of comment", "tag",
+ "end tag", "keyword", "keyword 2", "identifier", "literal", "string",
+ "character", "operator", "bracket", "separator", "url"
+ };
+}
initSymbolTable or
+ * lookup, if they are overridden. Symbols are inserted with
+ * symbolTable.put(sym,sym) and extracted with
+ * symbolTable.get(sym).
+ */
+ protected HashMapposition().
+ * Return the number of tokens successfully scanned, excluding any partial
+ * token at the end of the text segment but not at the end of the document. If
+ * the result is 0, the call should be retried with a longer segment.
+ */
+ public int scan(char[] array, int offset, int length) {
+ if (!scanning)
+ throw new Error("scan called when not scanning");
+ if (position + length > textLength)
+ throw new Error("scan too much");
+ boolean all = position + length == textLength;
+ end = start + length;
+ int startGap = gap;
+
+ buffer = array;
+ start = offset;
+ end = start + length;
+ while (start < end) {
+ int tokenStart = start;
+ int type = read();
+ if (start == end && !all)
+ break;
+
+ if (type != WHITESPACE) {
+ String name = new String(buffer, tokenStart, start - tokenStart);
+ Symbol sym = lookup(type, name);
+ Token t = new Token(sym, position);
+ if (gap >= endgap)
+ checkCapacity(gap + tokens.length - endgap + 1);
+ tokens[gap++] = t;
+ }
+
+ // Try to synchronise
+
+ while (tokens[endgap].position + textLength < position)
+ endgap++;
+ if (position + start - tokenStart == textLength)
+ scanning = false;
+ else if (gap > 0 && tokens[endgap].position + textLength == position
+ && tokens[endgap].symbol.type == type) {
+ endgap++;
+ scanning = false;
+ break;
+ }
+ position += start - tokenStart;
+ }
+ checkCapacity(gap + tokens.length - endgap);
+ return gap - startGap;
+ }
+
+ // Change the size of the gap buffer, doubling it if it fills up, and
+ // halving if it becomes less than a quarter full.
+
+ private void checkCapacity(int capacity) {
+ int oldCapacity = tokens.length;
+ if (capacity <= oldCapacity && 4 * capacity >= oldCapacity)
+ return;
+ Token[] oldTokens = tokens;
+ int newCapacity;
+ if (capacity > oldCapacity) {
+ newCapacity = oldCapacity * 2;
+ if (newCapacity < capacity)
+ newCapacity = capacity;
+ } else
+ newCapacity = capacity * 2;
+
+ tokens = new Token[newCapacity];
+ System.arraycopy(oldTokens, 0, tokens, 0, gap);
+ int n = oldCapacity - endgap;
+ System.arraycopy(oldTokens, endgap, tokens, newCapacity - n, n);
+ endgap = newCapacity - n;
+ }
+
+ void print() {
+ for (int i = 0; i < tokens.length; i++) {
+ if (i >= gap && i < endgap)
+ continue;
+ if (i == endgap)
+ System.out.print("... ");
+ System.out.print("" + i + ":" + tokens[i].position);
+ System.out.print("-"
+ + (tokens[i].position + tokens[i].symbol.name.length()));
+ System.out.print(" ");
+ }
+ System.out.println();
+ }
+}
diff --git a/se/sics/mspsim/extutil/highlight/Symbol.java b/se/sics/mspsim/extutil/highlight/Symbol.java
new file mode 100644
index 0000000..e957ab8
--- /dev/null
+++ b/se/sics/mspsim/extutil/highlight/Symbol.java
@@ -0,0 +1,51 @@
+package se.sics.mspsim.extutil.highlight;
+// Public domain, no restrictions, Ian Holyer, University of Bristol.
+
+/**
+ * A Symbol represents the information shared between similar tokens, i.e. their
+ * type and spelling.
+ */
+public class Symbol {
+ /**
+ * The type is a small integer used to classify symbols. It also distinguishes
+ * different symbols with the same spelling, where necessary.
+ */
+ public int type;
+
+ /**
+ * The spelling.
+ */
+ public String name;
+
+ /**
+ * Construct a symbol from its type and name.
+ */
+ public Symbol(int type, String name) {
+ this.type = type;
+ this.name = name;
+ }
+
+ /**
+ * Return the name of the symbol.
+ */
+ public String toString() {
+ return name;
+ }
+
+ /**
+ * Form a hash value from the type and name.
+ */
+ public int hashCode() {
+ return name.hashCode() + type;
+ }
+
+ /**
+ * Compare the type and name with some other symbol.
+ */
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Symbol))
+ return false;
+ Symbol that = (Symbol) obj;
+ return name.equals(that.name) && type == that.type;
+ }
+}
diff --git a/se/sics/mspsim/extutil/highlight/SyntaxHighlighter.java b/se/sics/mspsim/extutil/highlight/SyntaxHighlighter.java
new file mode 100644
index 0000000..cf649f7
--- /dev/null
+++ b/se/sics/mspsim/extutil/highlight/SyntaxHighlighter.java
@@ -0,0 +1,239 @@
+package se.sics.mspsim.extutil.highlight;
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.text.*;
+import javax.swing.event.*;
+import java.io.*;
+
+// Public domain, no restrictions, Ian Holyer, University of Bristol.
+
+/**
+ * Display text with syntax highlighting. Highlighting is done with full
+ * accuracy, using a given language scanner. Large amounts of re-highlighting
+ * are done in small bursts to make sure the user interface doesn't freeze.
+ */
+public class SyntaxHighlighter extends JTextPane implements DocumentListener, TokenTypes {
+
+
+ private static final long serialVersionUID = -1801145479677890566L;
+
+ private StyledDocument doc;
+ private Scanner scanner;
+ private int rows, columns;
+
+ /**
+ * Create a graphics component which displays text with syntax highlighting.
+ * Provide a width and height, in characters, and a language scanner.
+ */
+ public SyntaxHighlighter(int rows, int columns, Scanner scanner) {
+ super(new DefaultStyledDocument());
+ doc = (StyledDocument) getDocument();
+ this.rows = rows;
+ this.columns = columns;
+ this.scanner = scanner;
+ doc.addDocumentListener(this);
+ Font font = new Font("Monospaced", Font.PLAIN, getFont().getSize());
+ changeFont(font);
+ initStyles();
+ }
+
+ public int getColumns() {
+ return columns;
+ }
+
+ public int getRows() {
+ return rows;
+ }
+
+ /**
+ * Change the component's font, and change the size of the component to match.
+ */
+ public void changeFont(Font font) {
+ int borderOfJTextPane = 3;
+ setFont(font);
+ FontMetrics metrics = getFontMetrics(font);
+ int paneWidth = columns * metrics.charWidth('m') + 2 * borderOfJTextPane;
+ int paneHeight = rows * metrics.getHeight() + 2 * borderOfJTextPane;
+ Dimension size = new Dimension(paneWidth, paneHeight);
+ setMinimumSize(size);
+ setPreferredSize(size);
+ invalidate();
+ }
+
+ /**
+ * Read new text into the component from a Reader. Overrides
+ * read in JTextComponent in order to highlight
+ * the new text.
+ */
+ public void read(Reader in, Object desc) throws IOException {
+ int oldLength = getDocument().getLength();
+ doc.removeDocumentListener(this);
+ super.read(in, desc);
+ doc = (StyledDocument) getDocument();
+ doc.addDocumentListener(this);
+ int newLength = getDocument().getLength();
+ firstRehighlightToken = scanner.change(0, oldLength, newLength);
+ repaint();
+ }
+
+ // An array of styles, indexed by token type. Default styles are set up,
+ // which can be used for any languages.
+
+ private Style[] styles;
+
+ private void initStyles() {
+ styles = new Style[typeNames.length];
+ changeStyle(UNRECOGNIZED, Color.red);
+ changeStyle(WHITESPACE, Color.black);
+ changeStyle(WORD, Color.black);
+ changeStyle(NUMBER, Color.orange.darker());
+ changeStyle(PUNCTUATION, Color.orange.darker());
+ changeStyle(COMMENT, Color.green.darker());
+ changeStyle(START_COMMENT, Color.green.darker());
+ changeStyle(MID_COMMENT, Color.green.darker());
+ changeStyle(END_COMMENT, Color.green.darker());
+ changeStyle(TAG, Color.blue, Font.BOLD);
+ changeStyle(END_TAG, Color.blue, Font.BOLD);
+ changeStyle(KEYWORD, Color.blue);
+ changeStyle(KEYWORD2, Color.blue);
+ changeStyle(IDENTIFIER, Color.black);
+ changeStyle(LITERAL, Color.magenta);
+ changeStyle(STRING, Color.magenta);
+ changeStyle(CHARACTER, Color.magenta);
+ changeStyle(OPERATOR, Color.black, Font.BOLD);
+ changeStyle(BRACKET, Color.orange.darker());
+ changeStyle(SEPARATOR, Color.orange.darker());
+ changeStyle(URL, Color.blue.darker());
+
+ for (int i = 0; i < styles.length; i++) {
+ if (styles[i] == null) {
+ styles[i] = styles[WHITESPACE];
+ }
+ }
+ }
+
+ /**
+ * Change the style of a particular type of token.
+ */
+ public void changeStyle(int type, Color color) {
+ Style style = addStyle(typeNames[type], null);
+ StyleConstants.setForeground(style, color);
+ styles[type] = style;
+ }
+
+ /**
+ * Change the style of a particular type of token, including adding bold or
+ * italic using a third argument of Font.BOLD or
+ * Font.ITALIC or the bitwise union
+ * Font.BOLD|Font.ITALIC.
+ */
+ public void changeStyle(int type, Color color, int fontStyle) {
+ Style style = addStyle(typeNames[type], null);
+ StyleConstants.setForeground(style, color);
+ if ((fontStyle & Font.BOLD) != 0)
+ StyleConstants.setBold(style, true);
+ if ((fontStyle & Font.ITALIC) != 0)
+ StyleConstants.setItalic(style, true);
+ styles[type] = style;
+ }
+
+ public int getLineCount() {
+ return getDocument().getDefaultRootElement().getElementCount();
+ }
+
+ public int getLineStartOffset(int line) {
+ Element root = getDocument().getDefaultRootElement();
+ if (line < 0 || line >= root.getElementCount()) {
+ throw new IndexOutOfBoundsException("illegal line");
+ }
+ return root.getElement(line).getStartOffset();
+ }
+
+ public int getLineEndOffset(int line) {
+ Element root = getDocument().getDefaultRootElement();
+ if (line < 0 || line >= root.getElementCount()) {
+ throw new IndexOutOfBoundsException("illegal line");
+ }
+ return root.getElement(line).getEndOffset();
+ }
+
+ /**
+ * Ignore this method. Responds to the underlying
+ * document changes by re-highlighting.
+ */
+ public void insertUpdate(DocumentEvent e) {
+ int offset = e.getOffset();
+ int length = e.getLength();
+ firstRehighlightToken = scanner.change(offset, 0, length);
+ repaint();
+ }
+
+ /**
+ * Ignore this method. Responds to the underlying
+ * document changes by re-highlighting.
+ */
+ public void removeUpdate(DocumentEvent e) {
+ int offset = e.getOffset();
+ int length = e.getLength();
+ firstRehighlightToken = scanner.change(offset, length, 0);
+ repaint();
+ }
+
+ /**
+ * Ignore this method. Responds to the underlying
+ * document changes by re-highlighting.
+ */
+ public void changedUpdate(DocumentEvent e) {
+ // Do nothing.
+ }
+
+ // Scan a small portion of the document. If more is needed, call repaint()
+ // so the GUI gets a go and doesn't freeze, but calls this again later.
+
+ private Segment text = new Segment();
+
+ private int firstRehighlightToken;
+
+ private int smallAmount = 100;
+
+ /**
+ * Ignore this method. Carries out a small amount of
+ * re-highlighting for each call to repaint.
+ */
+ protected void paintComponent(Graphics g) {
+ super.paintComponent(g);
+ int offset = scanner.position();
+ if (offset < 0)
+ return;
+
+ int tokensToRedo = 0;
+ int amount = smallAmount;
+ while (tokensToRedo == 0 && offset >= 0) {
+ int length = doc.getLength() - offset;
+ if (length > amount) {
+ length = amount;
+ }
+ try {
+ doc.getText(offset, length, text);
+ } catch (BadLocationException e) {
+ return;
+ }
+ tokensToRedo = scanner.scan(text.array, text.offset, text.count);
+ offset = scanner.position();
+ amount = 2 * amount;
+ }
+ for (int i = 0; i < tokensToRedo; i++) {
+ Token t = scanner.getToken(firstRehighlightToken + i);
+ int length = t.symbol.name.length();
+ int type = t.symbol.type;
+ if (type < 0) {
+ type = UNRECOGNIZED;
+ }
+ doc.setCharacterAttributes(t.position, length, styles[type], false);
+ }
+ firstRehighlightToken += tokensToRedo;
+ if (offset >= 0) {
+ repaint(2);
+ }
+ }
+}
diff --git a/se/sics/mspsim/extutil/highlight/TextScanner.java b/se/sics/mspsim/extutil/highlight/TextScanner.java
new file mode 100644
index 0000000..677763f
--- /dev/null
+++ b/se/sics/mspsim/extutil/highlight/TextScanner.java
@@ -0,0 +1,62 @@
+package se.sics.mspsim.extutil.highlight;
+// This does exactly the same job as the Scanner class, but uses a table driven
+// scanner. This illustrates how to extend the Scanner class, and how a
+// generator might produce a table-driven scanner (though this one was written
+// by hand). Public domain, no restrictions, Ian Holyer, University of Bristol.
+
+public class TextScanner extends Scanner {
+ // Classify the 128 ASCII characters. Define a compact 'readable' form,
+ // then expand into an array, using static initializer code. The 128 are:
+ // .........tn..................... !"#$%&'()*+,-./0123456789:;<=>?
+ // @ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~.
+
+ static final String cs =
+ "1111111112211111111111111111111127777774777774675555555555777777"
+ + "7333333333333333333333333337777473333333333333333333333333377771";
+
+ static final byte[] kinds = new byte[cs.length()];
+ {
+ for (int i = 0; i < cs.length(); i++) {
+ kinds[i] = (byte) (cs.charAt(i) - '0');
+ }
+ }
+
+ // This state table has a row per scanner state, and a column per class of
+ // character. Each entry is '-s' to mean accept the current character and
+ // goto state s, or '+t' to mean end the token giving it type t. There is
+ // no stack of states, or extra table to determine the next state after a
+ // token is recognized, as there could be with more sophisticated scanners.
+ // The raw numbers +t must use or match the constants in TokenTypes.
+
+ int[][] table = {
+ // $ \0 \n z '-_ 9 . ( // $ = end of text, must be column 0
+ { +1, -1, -2, -3, -4, -5, -4, -4 }, // s=0: start token
+ { +1, +1, +1, +1, +1, +1, +1, +1 }, // s=1: illegal character (t=1)
+ { +0, +0, -2, +0, +0, +0, +0, +0 }, // s=2: whitespace (t=0)
+ { +2, +2, +2, -3, -3, -3, +2, +2 }, // s=3: word (t=2)
+ { +4, +4, +4, +4, +4, +4, +4, +4 }, // s=4: punctuation (t=4)
+ { +3, +3, +3, +3, +3, -5, -5, +3 } // s=5: number (t=3)
+ };
+
+ // Now all we have to do is to override read(), using the parent variables
+ // buffer, start, end, state.
+
+ protected int read() {
+ state = 0;
+ int kind = kinds[buffer[start]];
+ int type;
+ while (true) {
+ type = table[state][kind];
+ if (type >= 0)
+ break;
+ state = -type;
+ start++;
+ if (start >= end)
+ kind = 0;
+ else
+ kind = kinds[buffer[start]];
+ }
+ state = WHITESPACE;
+ return type;
+ }
+}
diff --git a/se/sics/mspsim/extutil/highlight/Token.java b/se/sics/mspsim/extutil/highlight/Token.java
new file mode 100644
index 0000000..75c6705
--- /dev/null
+++ b/se/sics/mspsim/extutil/highlight/Token.java
@@ -0,0 +1,26 @@
+package se.sics.mspsim.extutil.highlight;
+// Public domain, no restrictions, Ian Holyer, University of Bristol.
+
+/**
+ * A token represents a smallest meaningful fragment of text, such as a word,
+ * recognised by a scanner.
+ */
+public class Token {
+ /**
+ * The symbol contains all the properties shared with similar tokens.
+ */
+ public Symbol symbol;
+
+ /**
+ * The token's position is given by an index into the document text.
+ */
+ public int position;
+
+ /**
+ * Create a token with a given symbol and position.
+ */
+ public Token(Symbol symbol, int position) {
+ this.symbol = symbol;
+ this.position = position;
+ }
+}
diff --git a/se/sics/mspsim/extutil/highlight/TokenTypes.java b/se/sics/mspsim/extutil/highlight/TokenTypes.java
new file mode 100644
index 0000000..21a4f88
--- /dev/null
+++ b/se/sics/mspsim/extutil/highlight/TokenTypes.java
@@ -0,0 +1,50 @@
+package se.sics.mspsim.extutil.highlight;
+// Public domain, no restrictions, Ian Holyer, University of Bristol.
+
+/**
+ * The TokenTypes interface defines the integer constants representing different
+ * types of tokens, for use with any languages. The constants are used in
+ * symbols to represent the types of similar tokens, and in scanners as scanner
+ * states, and in highlighters to determine the colour or style of tokens. There
+ * is also an array typeNames of textual names, indexed by type, for descriptive
+ * purposes.
+ *
+ *