sdk/runtime/tools/wiki/build/cpp_indexer.py

# Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
#
"""Clang based C++ code indexer which produces xref.json."""
from __future__ import annotations

import glob
import json
import logging
import os
import platform
import re
import sys
import subprocess

from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Dict, Optional, Set

from marshmallow_dataclass import class_schema
from progress.bar import ShadyBar

_libclang_path = None
_clang_include_dir = None
if platform.system() == 'Darwin':
    _path_to_llvm = subprocess.check_output(['brew', '--prefix', 'llvm'],
                                            encoding='utf-8').strip()
    _clang_include_dir = glob.glob(f'{_path_to_llvm}/lib/clang/**/include')[0]
    _libclang_path = f'{_path_to_llvm}/lib/libclang.dylib'
    sys.path.append(
        f'{_path_to_llvm}/lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages'
    )

# pylint: disable=wrong-import-position,line-too-long
from clang.cindex import Config, CompilationDatabase, Cursor, CursorKind, Index, SourceLocation, TranslationUnit

if _libclang_path is not None:
    Config.set_library_file(_libclang_path)

_DART_CONFIGURATION = 'Release' + (
    'ARM64' if platform.uname().machine == 'arm64' else 'X64')
_DART_BUILD_DIR = ('xcodebuild' if platform.system() == 'Darwin' else
                   'out') + '/' + _DART_CONFIGURATION


@contextmanager
def _change_working_directory_to(path):
    oldpwd = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(oldpwd)


def _create_compilation_database():
    """Create compilation database for the default configuration.

    Extacts compilation database (compile_commands.json) for the default
    build configuration and filters it down to commands taht builds just
    the core VM pieces in the host configuration.
    """
    logging.info('Extracting compilation commands from build files for %s',
                 _DART_CONFIGURATION)
    with _change_working_directory_to(_DART_BUILD_DIR):
        commands = json.loads(
            subprocess.check_output(['ninja', '-C', '.', '-t', 'compdb',
                                     'cxx']))
    pattern = re.compile(
        r'libdart(_vm|_compiler)?_precompiler_host_targeting_host\.')
    with open('compile_commands.json', 'w', encoding='utf-8') as outfile:
        json.dump([
            cmd for cmd in commands
            if pattern.search(cmd['command']) is not None
        ], outfile)


def _get_current_commit_hash():
    return subprocess.check_output(['git', 'merge-base', 'main', 'HEAD'],
                                   text=True).strip()


@dataclass
class _ClassInfo:
    location: str
    members: Dict[str, str] = field(default_factory=dict)


@dataclass
class Location:
    """Symbol location referring to a line in a specific file."""
    filename: str
    lineno: int


@dataclass
class SymbolsIndex:
    """Index of C++ symbols extracted from source."""
    commit: str
    files: list[str] = field(default_factory=list)
    classes: Dict[str, _ClassInfo] = field(default_factory=dict)
    functions: Dict[str, str] = field(default_factory=dict)

    def try_resolve(self, ref: str) -> Optional[Location]:
        """Resolve the location of the given reference."""
        loc = self._try_resolve_impl(ref)
        if loc is not None:
            return self._location_from_string(loc)
        return None

    def _try_resolve_impl(self, ref: str) -> Optional[str]:
        if ref in self.functions:
            return self.functions[ref]
        if ref in self.classes:
            return self.classes[ref].location
        if '::' in ref:
            (class_name, function_name) = ref.rsplit('::', 1)
            if class_name in self.classes:
                return self.classes[class_name].members.get(function_name)
        return None

    def _location_from_string(self, loc: str) -> Location:
        (file_idx, line_idx) = loc.split(':', 1)
        return Location(self.files[int(file_idx)], int(line_idx))


class _Indexer:
    symbols_index: SymbolsIndex

    processed_files: Set[str]
    classes_by_usr: Dict[str, _ClassInfo]

    name_stack: list[str]
    info_stack: list[_ClassInfo]
    files_seen_in_unit: Set[str]

    def __init__(self):
        self.symbols_index = SymbolsIndex(commit=_get_current_commit_hash())

        self.processed_files = set()
        self.classes_by_usr = {}
        self.files_index = {}

        self.name_stack = []
        self.info_stack = []
        self.files_seen_in_unit = set()

    def index(self, unit: TranslationUnit):
        """Index the given translation unit and append new symbols to index."""
        self.name_stack.clear()
        self.info_stack.clear()
        self.files_seen_in_unit.clear()
        self._recurse(unit.cursor)
        self.processed_files |= self.files_seen_in_unit

    def _recurse(self, cursor: Cursor):
        name = ""
        kind = cursor.kind

        if cursor.location.file is not None:
            name = cursor.location.file.name
            if name in self.processed_files or not name.startswith('../..'):
                return
            self.files_seen_in_unit.add(name)

        if kind == CursorKind.CLASS_DECL:
            if not cursor.is_definition():
                return
            usr = cursor.get_usr()
            if usr in self.classes_by_usr:
                return
            self.name_stack.append(cursor.spelling)
            class_name = '::'.join(self.name_stack)
            class_info = _ClassInfo(self._format_location(cursor.location))
            self.info_stack.append(class_info)
            self.symbols_index.classes[class_name] = class_info
            self.classes_by_usr[usr] = class_info
        elif kind == CursorKind.NAMESPACE:
            self.name_stack.append(cursor.spelling)
        elif kind == CursorKind.FUNCTION_DECL and cursor.is_definition():
            namespace_prefix = ""
            if cursor.semantic_parent.kind == CursorKind.NAMESPACE:
                namespace_prefix = '::'.join(self.name_stack) + (
                    '::' if len(self.name_stack) > 0 else "")
            function_name = namespace_prefix + cursor.spelling
            self.symbols_index.functions[function_name] = self._format_location(
                cursor.location)
            return
        elif kind == CursorKind.CXX_METHOD and cursor.is_definition():
            parent = cursor.semantic_parent
            if parent.kind == CursorKind.CLASS_DECL:
                class_info_or_none = self.classes_by_usr.get(parent.get_usr())
                if class_info_or_none is None:
                    return
                class_info_or_none.members[
                    cursor.spelling] = self._format_location(cursor.location)
            return
        elif kind == CursorKind.VAR_DECL and cursor.is_definition():
            parent = cursor.semantic_parent
            if parent.kind == CursorKind.CLASS_DECL:
                class_info_or_none = self.classes_by_usr.get(parent.get_usr())
                if class_info_or_none is None:
                    return
                class_info_or_none.members[
                    cursor.spelling] = self._format_location(cursor.location)

        for child in cursor.get_children():
            self._recurse(child)

        if kind == CursorKind.NAMESPACE:
            self.name_stack.pop()
        elif kind == CursorKind.CLASS_DECL:
            self.name_stack.pop()
            self.info_stack.pop()

    def _format_location(self, loc: SourceLocation):
        file_name = loc.file.name
        lineno = loc.line
        return f'{self._get_file_index(file_name)}:{lineno}'

    def _get_file_index(self, file_name: str):
        index = self.files_index.get(file_name)
        if index is None:
            index = len(self.symbols_index.files)
            self.files_index[file_name] = index
            self.symbols_index.files.append(
                os.path.relpath(os.path.abspath(file_name),
                                os.path.abspath('../..')))
        return index


def _index_source() -> SymbolsIndex:
    indexer = _Indexer()

    _create_compilation_database()
    with _change_working_directory_to(_DART_BUILD_DIR):
        index = Index.create()
        compdb = CompilationDatabase.fromDirectory('.')

        commands = list(compdb.getAllCompileCommands())
        with ShadyBar('Indexing',
                      max=len(commands),
                      suffix='%(percent)d%% eta %(eta_td)s') as progress_bar:
            for command in commands:
                args = [
                    arg for arg in command.arguments
                    if arg.startswith('-I') or arg.startswith('-W') or
                    arg.startswith('-D') or arg.startswith('-i') or
                    arg.startswith('sdk/') or arg.startswith('-std')
                ] + [
                    '-Wno-macro-redefined', '-Wno-unused-const-variable',
                    '-Wno-unused-function', '-Wno-unused-variable'
                ]

                if _clang_include_dir is not None:
                    args.append(f'-I{_clang_include_dir}')

                unit = index.parse(command.filename, args=args)
                for diag in unit.diagnostics:
                    print(diag.format())

                indexer.index(unit)
                progress_bar.next()

    return indexer.symbols_index


_SymbolsIndexSchema = class_schema(SymbolsIndex)()


def load_index(filename: str) -> SymbolsIndex:
    """Load symbols index from the given file.

    If index is out of date or missing it will be generated.
    """
    index: SymbolsIndex

    if os.path.exists(filename):
        with open(filename, 'r', encoding='utf-8') as json_file:
            index = _SymbolsIndexSchema.loads(json_file.read())
        if _get_current_commit_hash() == index.commit:
            logging.info('Loaded symbols index from %s', filename)
            return index
        logging.warning(
            '%s is generated for commit %s while current commit is %s',
            filename, index.commit, _get_current_commit_hash())

    index = _index_source()
    with open(filename, 'w', encoding='utf-8') as json_file:
        json_file.write(_SymbolsIndexSchema.dumps(index))
    logging.info(
        'Successfully indexed C++ source and written symbols index into %s',
        filename)
    return index