| #!/usr/bin/env python |
| # |
| # Copyright 2007 Neal Norwitz |
| # Portions Copyright 2007 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Tokenize C++ source code.""" |
| |
| __author__ = 'nnorwitz@google.com (Neal Norwitz)' |
| |
| |
| try: |
| # Python 3.x |
| import builtins |
| except ImportError: |
| # Python 2.x |
| import __builtin__ as builtins |
| |
| |
| import sys |
| |
| from cpp import utils |
| |
| |
| if not hasattr(builtins, 'set'): |
| # Nominal support for Python 2.3. |
| from sets import Set as set |
| |
| |
| # Add $ as a valid identifier char since so much code uses it. |
| _letters = 'abcdefghijklmnopqrstuvwxyz' |
| VALID_IDENTIFIER_CHARS = set(_letters + _letters.upper() + '_0123456789$') |
| HEX_DIGITS = set('0123456789abcdefABCDEF') |
| INT_OR_FLOAT_DIGITS = set('01234567890eE-+') |
| |
| |
| # C++0x string preffixes. |
| _STR_PREFIXES = set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR')) |
| |
| |
| # Token types. |
| UNKNOWN = 'UNKNOWN' |
| SYNTAX = 'SYNTAX' |
| CONSTANT = 'CONSTANT' |
| NAME = 'NAME' |
| PREPROCESSOR = 'PREPROCESSOR' |
| |
| # Where the token originated from. This can be used for backtracking. |
| # It is always set to WHENCE_STREAM in this code. |
| WHENCE_STREAM, WHENCE_QUEUE = range(2) |
| |
| |
| class Token(object): |
| """Data container to represent a C++ token. |
| |
| Tokens can be identifiers, syntax char(s), constants, or |
| pre-processor directives. |
| |
| start contains the index of the first char of the token in the source |
| end contains the index of the last char of the token in the source |
| """ |
| |
| def __init__(self, token_type, name, start, end): |
| self.token_type = token_type |
| self.name = name |
| self.start = start |
| self.end = end |
| self.whence = WHENCE_STREAM |
| |
| def __str__(self): |
| if not utils.DEBUG: |
| return 'Token(%r)' % self.name |
| return 'Token(%r, %s, %s)' % (self.name, self.start, self.end) |
| |
| __repr__ = __str__ |
| |
| |
| def _GetString(source, start, i): |
| i = source.find('"', i+1) |
| while source[i-1] == '\\': |
| # Count the trailing backslashes. |
| backslash_count = 1 |
| j = i - 2 |
| while source[j] == '\\': |
| backslash_count += 1 |
| j -= 1 |
| # When trailing backslashes are even, they escape each other. |
| if (backslash_count % 2) == 0: |
| break |
| i = source.find('"', i+1) |
| return i + 1 |
| |
| |
| def _GetChar(source, start, i): |
| # NOTE(nnorwitz): may not be quite correct, should be good enough. |
| i = source.find("'", i+1) |
| while source[i-1] == '\\': |
| # Need to special case '\\'. |
| if (i - 2) > start and source[i-2] == '\\': |
| break |
| i = source.find("'", i+1) |
| # Try to handle unterminated single quotes (in a #if 0 block). |
| if i < 0: |
| i = start |
| return i + 1 |
| |
| |
| def GetTokens(source): |
| """Returns a sequence of Tokens. |
| |
| Args: |
| source: string of C++ source code. |
| |
| Yields: |
| Token that represents the next token in the source. |
| """ |
| # Cache various valid character sets for speed. |
| valid_identifier_chars = VALID_IDENTIFIER_CHARS |
| hex_digits = HEX_DIGITS |
| int_or_float_digits = INT_OR_FLOAT_DIGITS |
| int_or_float_digits2 = int_or_float_digits | set('.') |
| |
| # Only ignore errors while in a #if 0 block. |
| ignore_errors = False |
| count_ifs = 0 |
| |
| i = 0 |
| end = len(source) |
| while i < end: |
| # Skip whitespace. |
| while i < end and source[i].isspace(): |
| i += 1 |
| if i >= end: |
| return |
| |
| token_type = UNKNOWN |
| start = i |
| c = source[i] |
| if c.isalpha() or c == '_': # Find a string token. |
| token_type = NAME |
| while source[i] in valid_identifier_chars: |
| i += 1 |
| # String and character constants can look like a name if |
| # they are something like L"". |
| if (source[i] == "'" and (i - start) == 1 and |
| source[start:i] in 'uUL'): |
| # u, U, and L are valid C++0x character preffixes. |
| token_type = CONSTANT |
| i = _GetChar(source, start, i) |
| elif source[i] == "'" and source[start:i] in _STR_PREFIXES: |
| token_type = CONSTANT |
| i = _GetString(source, start, i) |
| elif c == '/' and source[i+1] == '/': # Find // comments. |
| i = source.find('\n', i) |
| if i == -1: # Handle EOF. |
| i = end |
| continue |
| elif c == '/' and source[i+1] == '*': # Find /* comments. */ |
| i = source.find('*/', i) + 2 |
| continue |
| elif c in ':+-<>&|*=': # : or :: (plus other chars). |
| token_type = SYNTAX |
| i += 1 |
| new_ch = source[i] |
| if new_ch == c and c != '>': # Treat ">>" as two tokens. |
| i += 1 |
| elif c == '-' and new_ch == '>': |
| i += 1 |
| elif new_ch == '=': |
| i += 1 |
| elif c in '()[]{}~!?^%;/.,': # Handle single char tokens. |
| token_type = SYNTAX |
| i += 1 |
| if c == '.' and source[i].isdigit(): |
| token_type = CONSTANT |
| i += 1 |
| while source[i] in int_or_float_digits: |
| i += 1 |
| # Handle float suffixes. |
| for suffix in ('l', 'f'): |
| if suffix == source[i:i+1].lower(): |
| i += 1 |
| break |
| elif c.isdigit(): # Find integer. |
| token_type = CONSTANT |
| if c == '0' and source[i+1] in 'xX': |
| # Handle hex digits. |
| i += 2 |
| while source[i] in hex_digits: |
| i += 1 |
| else: |
| while source[i] in int_or_float_digits2: |
| i += 1 |
| # Handle integer (and float) suffixes. |
| for suffix in ('ull', 'll', 'ul', 'l', 'f', 'u'): |
| size = len(suffix) |
| if suffix == source[i:i+size].lower(): |
| i += size |
| break |
| elif c == '"': # Find string. |
| token_type = CONSTANT |
| i = _GetString(source, start, i) |
| elif c == "'": # Find char. |
| token_type = CONSTANT |
| i = _GetChar(source, start, i) |
| elif c == '#': # Find pre-processor command. |
| token_type = PREPROCESSOR |
| got_if = source[i:i+3] == '#if' and source[i+3:i+4].isspace() |
| if got_if: |
| count_ifs += 1 |
| elif source[i:i+6] == '#endif': |
| count_ifs -= 1 |
| if count_ifs == 0: |
| ignore_errors = False |
| |
| # TODO(nnorwitz): handle preprocessor statements (\ continuations). |
| while 1: |
| i1 = source.find('\n', i) |
| i2 = source.find('//', i) |
| i3 = source.find('/*', i) |
| i4 = source.find('"', i) |
| # NOTE(nnorwitz): doesn't handle comments in #define macros. |
| # Get the first important symbol (newline, comment, EOF/end). |
| i = min([x for x in (i1, i2, i3, i4, end) if x != -1]) |
| |
| # Handle #include "dir//foo.h" properly. |
| if source[i] == '"': |
| i = source.find('"', i+1) + 1 |
| assert i > 0 |
| continue |
| # Keep going if end of the line and the line ends with \. |
| if not (i == i1 and source[i-1] == '\\'): |
| if got_if: |
| condition = source[start+4:i].lstrip() |
| if (condition.startswith('0') or |
| condition.startswith('(0)')): |
| ignore_errors = True |
| break |
| i += 1 |
| elif c == '\\': # Handle \ in code. |
| # This is different from the pre-processor \ handling. |
| i += 1 |
| continue |
| elif ignore_errors: |
| # The tokenizer seems to be in pretty good shape. This |
| # raise is conditionally disabled so that bogus code |
| # in an #if 0 block can be handled. Since we will ignore |
| # it anyways, this is probably fine. So disable the |
| # exception and return the bogus char. |
| i += 1 |
| else: |
| sys.stderr.write('Got invalid token in %s @ %d token:%s: %r\n' % |
| ('?', i, c, source[i-10:i+10])) |
| raise RuntimeError('unexpected token') |
| |
| if i <= 0: |
| print('Invalid index, exiting now.') |
| return |
| yield Token(token_type, source[start:i], start, i) |
| |
| |
| if __name__ == '__main__': |
| def main(argv): |
| """Driver mostly for testing purposes.""" |
| for filename in argv[1:]: |
| source = utils.ReadFile(filename) |
| if source is None: |
| continue |
| |
| for token in GetTokens(source): |
| print('%-12s: %s' % (token.token_type, token.name)) |
| # print('\r%6.2f%%' % (100.0 * index / token.end),) |
| sys.stdout.write('\n') |
| |
| |
| main(sys.argv) |
