util/find_copyrights.py

#!/usr/bin/env python3

import os
import re
import sys

from file_types import lang_type, find_files

mode_line = re.compile("(-\*- *mode:.* *-\*-)")
shell_comment = re.compile(r"^\s*#")
lisp_comment = re.compile(r";")
cpp_comment = re.compile(r"//")
c_comment_start = re.compile(r"/\*")
c_comment_end = re.compile(r"\*/")


def find_copyright_block(lines, lang_type):
    start = None
    if lang_type in ("python", "make", "shell", "perl", "scons"):
        for i, line in enumerate(lines):
            if i == 0 and (line.startswith("#!") or mode_line.search(line)):
                continue

            if shell_comment.search(line):
                if start is None:
                    start = i
            elif start is None:
                if line.strip():
                    return
            else:
                yield start, i - 1
                start = None

    elif lang_type in ("lisp",):
        for i, line in enumerate(lines):
            if i == 0 and mode_line.search(line):
                continue

            if lisp_comment.search(line):
                if start is None:
                    start = i
            elif start is None:
                if line.strip():
                    return
            else:
                yield start, i - 1
                start = None

    elif lang_type in (
        "C",
        "C++",
        "swig",
        "isa",
        "asm",
        "slicc",
        "lex",
        "yacc",
    ):
        mode = None
        for i, line in enumerate(lines):
            if i == 0 and mode_line.search(line):
                continue

            if mode == "C":
                assert start is not None, "on line %d" % (i + 1)
                match = c_comment_end.search(line)
                if match:
                    yield start, i
                    mode = None
                continue

            cpp_match = cpp_comment.search(line)
            c_match = c_comment_start.search(line)

            if cpp_match:
                assert not c_match, "on line %d" % (i + 1)
                if line[: cpp_match.start()].strip():
                    return
                if mode is None:
                    mode = "CPP"
                    start = i
                else:
                    text = line[cpp_match.end() :].lstrip()
                    if text.startswith("Copyright") > 0:
                        yield start, i - 1
                        start = i
                continue
            elif mode == "CPP":
                assert start is not None, "on line %d" % (i + 1)
                if not line.strip():
                    continue
                yield start, i - 1
                mode = None
                if not c_match:
                    return

            if c_match:
                assert mode is None, "on line %d" % (i + 1)
                mode = "C"
                start = i

            if mode is None and line.strip():
                return

    else:
        raise AttributeError("Could not handle language %s" % lang_type)


date_range_re = re.compile(r"([0-9]{4})\s*-\s*([0-9]{4})")


def process_dates(dates):
    dates = [d.strip() for d in dates.split(",")]

    output = set()
    for date in dates:
        match = date_range_re.match(date)
        if match:
            f, l = [int(d) for d in match.groups()]
            for i in range(f, l + 1):
                output.add(i)
        else:
            try:
                date = int(date)
                output.add(date)
            except ValueError:
                pass

    return output


copyright_re = re.compile(
    r"Copyright (\([cC]\)) ([-, 0-9]+)[\s*#/]*([A-z-,. ]+)", re.DOTALL
)

authors_re = re.compile(r"^[\s*#/]*Authors:\s*([A-z .]+)\s*$")
more_authors_re = re.compile(r"^[\s*#/]*([A-z .]+)\s*$")

all_owners = set()


def get_data(lang_type, lines):
    data = []
    last = None
    for start, end in find_copyright_block(lines, lang_type):
        joined = "".join(lines[start : end + 1])
        match = copyright_re.search(joined)
        if not match:
            continue

        c, dates, owner = match.groups()
        dates = dates.strip()
        owner = owner.strip()

        all_owners.add(owner)
        try:
            dates = process_dates(dates)
        except Exception:
            print(dates)
            print(owner)
            raise

        authors = []
        for i in range(start, end + 1):
            line = lines[i]
            if not authors:
                match = authors_re.search(line)
                if match:
                    authors.append(match.group(1).strip())
            else:
                match = more_authors_re.search(line)
                if not match:
                    for j in range(i, end + 1):
                        line = lines[j].strip()
                        if not line:
                            end = j
                            break
                        if line.startswith("//"):
                            line = line[2:].lstrip()
                            if line:
                                end = j - 1
                                break
                    break
                authors.append(match.group(1).strip())

        info = (owner, dates, authors, start, end)
        data.append(info)

    return data


def datestr(dates):
    dates = list(dates)
    dates.sort()

    output = []

    def add_output(first, second):
        if first == second:
            output.append("%d" % (first))
        else:
            output.append("%d-%d" % (first, second))

    first = dates.pop(0)
    second = first
    while dates:
        next = dates.pop(0)
        if next == second + 1:
            second = next
        else:
            add_output(first, second)
            first = next
            second = next

    add_output(first, second)

    return ",".join(output)


usage_str = """usage:
%s [-v] <directory>"""


def usage(exitcode):
    print(usage_str % sys.argv[0])
    if exitcode is not None:
        sys.exit(exitcode)


if __name__ == "__main__":
    import getopt

    show_counts = False
    ignore = set()
    verbose = False
    try:
        opts, args = getopt.getopt(sys.argv[1:], "ci:v")
    except getopt.GetoptError:
        usage(1)

    for o, a in opts:
        if o == "-c":
            show_counts = True
        if o == "-i":
            ignore.add(a)
        if o == "-v":
            verbose = True

    files = []

    for base in args:
        if os.path.isfile(base):
            files += [(base, lang_type(base))]
        elif os.path.isdir(base):
            files += find_files(base)
        else:
            raise AttributeError("can't access '%s'" % base)

    copyrights = {}
    counts = {}

    for filename, lang in files:
        f = file(filename, "r")
        lines = f.readlines()
        if not lines:
            continue

        lines = [line.rstrip("\r\n") for line in lines]

        lt = lang_type(filename, lines[0])
        try:
            data = get_data(lt, lines)
        except Exception as e:
            if verbose:
                if len(e.args) == 1:
                    e.args = ("%s (%s))" % (e, filename),)
                print("could not parse %s: %s" % (filename, e))
            continue

        for owner, dates, authors, start, end in data:
            if owner not in copyrights:
                copyrights[owner] = set()
            if owner not in counts:
                counts[owner] = 0

            copyrights[owner] |= dates
            counts[owner] += 1

    info = [(counts[o], d, o) for o, d in list(copyrights.items())]

    for count, dates, owner in sorted(info, reverse=True):
        if show_counts:
            owner = "%s (%s files)" % (owner, count)
        print("Copyright (c) %s %s" % (datestr(dates), owner))