| # ---------------------------------------------------------------------- |
| # ctokens.py |
| # |
| # Token specifications for symbols in ANSI C and C++. This file is |
| # meant to be used as a library in other tokenizers. |
| # ---------------------------------------------------------------------- |
| |
| # Reserved words |
| |
| tokens = [ |
| # Literals (identifier, integer constant, float constant, string constant, char const) |
| 'ID', 'TYPEID', 'ICONST', 'FCONST', 'SCONST', 'CCONST', |
| |
| # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) |
| 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', |
| 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', |
| 'LOR', 'LAND', 'LNOT', |
| 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', |
| |
| # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) |
| 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', |
| 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', |
| |
| # Increment/decrement (++,--) |
| 'PLUSPLUS', 'MINUSMINUS', |
| |
| # Structure dereference (->) |
| 'ARROW', |
| |
| # Ternary operator (?) |
| 'TERNARY', |
| |
| # Delimeters ( ) [ ] { } , . ; : |
| 'LPAREN', 'RPAREN', |
| 'LBRACKET', 'RBRACKET', |
| 'LBRACE', 'RBRACE', |
| 'COMMA', 'PERIOD', 'SEMI', 'COLON', |
| |
| # Ellipsis (...) |
| 'ELLIPSIS', |
| ] |
| |
| # Operators |
| t_PLUS = r'\+' |
| t_MINUS = r'-' |
| t_TIMES = r'\*' |
| t_DIVIDE = r'/' |
| t_MODULO = r'%' |
| t_OR = r'\|' |
| t_AND = r'&' |
| t_NOT = r'~' |
| t_XOR = r'\^' |
| t_LSHIFT = r'<<' |
| t_RSHIFT = r'>>' |
| t_LOR = r'\|\|' |
| t_LAND = r'&&' |
| t_LNOT = r'!' |
| t_LT = r'<' |
| t_GT = r'>' |
| t_LE = r'<=' |
| t_GE = r'>=' |
| t_EQ = r'==' |
| t_NE = r'!=' |
| |
| # Assignment operators |
| |
| t_EQUALS = r'=' |
| t_TIMESEQUAL = r'\*=' |
| t_DIVEQUAL = r'/=' |
| t_MODEQUAL = r'%=' |
| t_PLUSEQUAL = r'\+=' |
| t_MINUSEQUAL = r'-=' |
| t_LSHIFTEQUAL = r'<<=' |
| t_RSHIFTEQUAL = r'>>=' |
| t_ANDEQUAL = r'&=' |
| t_OREQUAL = r'\|=' |
| t_XOREQUAL = r'^=' |
| |
| # Increment/decrement |
| t_INCREMENT = r'\+\+' |
| t_DECREMENT = r'--' |
| |
| # -> |
| t_ARROW = r'->' |
| |
| # ? |
| t_TERNARY = r'\?' |
| |
| # Delimeters |
| t_LPAREN = r'\(' |
| t_RPAREN = r'\)' |
| t_LBRACKET = r'\[' |
| t_RBRACKET = r'\]' |
| t_LBRACE = r'\{' |
| t_RBRACE = r'\}' |
| t_COMMA = r',' |
| t_PERIOD = r'\.' |
| t_SEMI = r';' |
| t_COLON = r':' |
| t_ELLIPSIS = r'\.\.\.' |
| |
| # Identifiers |
| t_ID = r'[A-Za-z_][A-Za-z0-9_]*' |
| |
| # Integer literal |
| t_INTEGER = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?' |
| |
| # Floating literal |
| t_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' |
| |
| # String literal |
| t_STRING = r'\"([^\\\n]|(\\.))*?\"' |
| |
| # Character constant 'c' or L'c' |
| t_CHARACTER = r'(L)?\'([^\\\n]|(\\.))*?\'' |
| |
| # Comment (C-Style) |
| def t_COMMENT(t): |
| r'/\*(.|\n)*?\*/' |
| t.lexer.lineno += t.value.count('\n') |
| return t |
| |
| # Comment (C++-Style) |
| def t_CPPCOMMENT(t): |
| r'//.*\n' |
| t.lexer.lineno += 1 |
| return t |
| |
| |
| |
| |
| |
| |