# Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. # """Clang based C++ code indexer which produces xref.json.""" from __future__ import annotations import glob import json import logging import os import platform import re import sys import subprocess from contextlib import contextmanager from dataclasses import dataclass, field from typing import Dict, Optional, Set from marshmallow_dataclass import class_schema from progress.bar import ShadyBar _libclang_path = None _clang_include_dir = None if platform.system() == 'Darwin': _path_to_llvm = subprocess.check_output(['brew', '--prefix', 'llvm'], encoding='utf-8').strip() _clang_include_dir = glob.glob(f'{_path_to_llvm}/lib/clang/**/include')[0] _libclang_path = f'{_path_to_llvm}/lib/libclang.dylib' sys.path.append( f'{_path_to_llvm}/lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages' ) # pylint: disable=wrong-import-position,line-too-long from clang.cindex import Config, CompilationDatabase, Cursor, CursorKind, Index, SourceLocation, TranslationUnit if _libclang_path is not None: Config.set_library_file(_libclang_path) _DART_CONFIGURATION = 'Release' + ( 'ARM64' if platform.uname().machine == 'arm64' else 'X64') _DART_BUILD_DIR = ('xcodebuild' if platform.system() == 'Darwin' else 'out') + '/' + _DART_CONFIGURATION @contextmanager def _change_working_directory_to(path): oldpwd = os.getcwd() os.chdir(path) try: yield finally: os.chdir(oldpwd) def _create_compilation_database(): """Create compilation database for the default configuration. Extacts compilation database (compile_commands.json) for the default build configuration and filters it down to commands taht builds just the core VM pieces in the host configuration. """ logging.info('Extracting compilation commands from build files for %s', _DART_CONFIGURATION) with _change_working_directory_to(_DART_BUILD_DIR): commands = json.loads( subprocess.check_output(['ninja', '-C', '.', '-t', 'compdb', 'cxx'])) pattern = re.compile( r'libdart(_vm|_compiler)?_precompiler_host_targeting_host\.') with open('compile_commands.json', 'w', encoding='utf-8') as outfile: json.dump([ cmd for cmd in commands if pattern.search(cmd['command']) is not None ], outfile) def _get_current_commit_hash(): return subprocess.check_output(['git', 'merge-base', 'main', 'HEAD'], text=True).strip() @dataclass class _ClassInfo: location: str members: Dict[str, str] = field(default_factory=dict) @dataclass class Location: """Symbol location referring to a line in a specific file.""" filename: str lineno: int @dataclass class SymbolsIndex: """Index of C++ symbols extracted from source.""" commit: str files: list[str] = field(default_factory=list) classes: Dict[str, _ClassInfo] = field(default_factory=dict) functions: Dict[str, str] = field(default_factory=dict) def try_resolve(self, ref: str) -> Optional[Location]: """Resolve the location of the given reference.""" loc = self._try_resolve_impl(ref) if loc is not None: return self._location_from_string(loc) return None def _try_resolve_impl(self, ref: str) -> Optional[str]: if ref in self.functions: return self.functions[ref] if ref in self.classes: return self.classes[ref].location if '::' in ref: (class_name, function_name) = ref.rsplit('::', 1) if class_name in self.classes: return self.classes[class_name].members.get(function_name) return None def _location_from_string(self, loc: str) -> Location: (file_idx, line_idx) = loc.split(':', 1) return Location(self.files[int(file_idx)], int(line_idx)) class _Indexer: symbols_index: SymbolsIndex processed_files: Set[str] classes_by_usr: Dict[str, _ClassInfo] name_stack: list[str] info_stack: list[_ClassInfo] files_seen_in_unit: Set[str] def __init__(self): self.symbols_index = SymbolsIndex(commit=_get_current_commit_hash()) self.processed_files = set() self.classes_by_usr = {} self.files_index = {} self.name_stack = [] self.info_stack = [] self.files_seen_in_unit = set() def index(self, unit: TranslationUnit): """Index the given translation unit and append new symbols to index.""" self.name_stack.clear() self.info_stack.clear() self.files_seen_in_unit.clear() self._recurse(unit.cursor) self.processed_files |= self.files_seen_in_unit def _recurse(self, cursor: Cursor): name = "" kind = cursor.kind if cursor.location.file is not None: name = cursor.location.file.name if name in self.processed_files or not name.startswith('../..'): return self.files_seen_in_unit.add(name) if kind == CursorKind.CLASS_DECL: if not cursor.is_definition(): return usr = cursor.get_usr() if usr in self.classes_by_usr: return self.name_stack.append(cursor.spelling) class_name = '::'.join(self.name_stack) class_info = _ClassInfo(self._format_location(cursor.location)) self.info_stack.append(class_info) self.symbols_index.classes[class_name] = class_info self.classes_by_usr[usr] = class_info elif kind == CursorKind.NAMESPACE: self.name_stack.append(cursor.spelling) elif kind == CursorKind.FUNCTION_DECL and cursor.is_definition(): namespace_prefix = "" if cursor.semantic_parent.kind == CursorKind.NAMESPACE: namespace_prefix = '::'.join(self.name_stack) + ( '::' if len(self.name_stack) > 0 else "") function_name = namespace_prefix + cursor.spelling self.symbols_index.functions[function_name] = self._format_location( cursor.location) return elif kind == CursorKind.CXX_METHOD and cursor.is_definition(): parent = cursor.semantic_parent if parent.kind == CursorKind.CLASS_DECL: class_info_or_none = self.classes_by_usr.get(parent.get_usr()) if class_info_or_none is None: return class_info_or_none.members[ cursor.spelling] = self._format_location(cursor.location) return elif kind == CursorKind.VAR_DECL and cursor.is_definition(): parent = cursor.semantic_parent if parent.kind == CursorKind.CLASS_DECL: class_info_or_none = self.classes_by_usr.get(parent.get_usr()) if class_info_or_none is None: return class_info_or_none.members[ cursor.spelling] = self._format_location(cursor.location) for child in cursor.get_children(): self._recurse(child) if kind == CursorKind.NAMESPACE: self.name_stack.pop() elif kind == CursorKind.CLASS_DECL: self.name_stack.pop() self.info_stack.pop() def _format_location(self, loc: SourceLocation): file_name = loc.file.name lineno = loc.line return f'{self._get_file_index(file_name)}:{lineno}' def _get_file_index(self, file_name: str): index = self.files_index.get(file_name) if index is None: index = len(self.symbols_index.files) self.files_index[file_name] = index self.symbols_index.files.append( os.path.relpath(os.path.abspath(file_name), os.path.abspath('../..'))) return index def _index_source() -> SymbolsIndex: indexer = _Indexer() _create_compilation_database() with _change_working_directory_to(_DART_BUILD_DIR): index = Index.create() compdb = CompilationDatabase.fromDirectory('.') commands = list(compdb.getAllCompileCommands()) with ShadyBar('Indexing', max=len(commands), suffix='%(percent)d%% eta %(eta_td)s') as progress_bar: for command in commands: args = [ arg for arg in command.arguments if arg.startswith('-I') or arg.startswith('-W') or arg.startswith('-D') or arg.startswith('-i') or arg.startswith('sdk/') or arg.startswith('-std') ] + [ '-Wno-macro-redefined', '-Wno-unused-const-variable', '-Wno-unused-function', '-Wno-unused-variable' ] if _clang_include_dir is not None: args.append(f'-I{_clang_include_dir}') unit = index.parse(command.filename, args=args) for diag in unit.diagnostics: print(diag.format()) indexer.index(unit) progress_bar.next() return indexer.symbols_index _SymbolsIndexSchema = class_schema(SymbolsIndex)() def load_index(filename: str) -> SymbolsIndex: """Load symbols index from the given file. If index is out of date or missing it will be generated. """ index: SymbolsIndex if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as json_file: index = _SymbolsIndexSchema.loads(json_file.read()) if _get_current_commit_hash() == index.commit: logging.info('Loaded symbols index from %s', filename) return index logging.warning( '%s is generated for commit %s while current commit is %s', filename, index.commit, _get_current_commit_hash()) index = _index_source() with open(filename, 'w', encoding='utf-8') as json_file: json_file.write(_SymbolsIndexSchema.dumps(index)) logging.info( 'Successfully indexed C++ source and written symbols index into %s', filename) return index