[vm/docs] Improve VM docs tooling.
Main change is around removing our custom syntax which allows
to read markdown files directly on GitHub:
* instead of using custom link format `@{xref}` we start
using normal links. For example, [`dart::ThreadPool`][] is
understood as a ref to `dart::ThreadPool` class declaration.
`build.py` script injects an xref section at the end of
each markdown file.
* similarly we don't use custom syntax for admonitions, but
instead use blockquotes. `build.py` detects block quotes
which start with a marker like `**Note**` and renders
then in a custom way.
This CL also drops dependency on cquery and instead rewrites
indexing in pure Python via libclang.
Change-Id: I0b47ec93f632de89627a3c682d511c8b86c58430
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/280262
Reviewed-by: Martin Kustermann <kustermann@google.com>
Commit-Queue: Slava Egorov <vegorov@google.com>
This commit is contained in:
Vyacheslav Egorov
committed by
Commit Queue
Commit Queue
parent
763edcaf86
commit
b2d5245dcd
Binary file not shown.
|
Before Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 35 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 96 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 36 KiB |
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 76 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 89 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 54 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 35 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 176 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 17 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 38 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB |
+748
-181
File diff suppressed because it is too large
Load Diff
@@ -1,5 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
#
|
||||
@@ -1,49 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
#
|
||||
"""Simple lexer for shell sessions.
|
||||
|
||||
Highlights command lines (lines starting with $ prompt) and comments starting
|
||||
with #. For example:
|
||||
|
||||
# This is a comment
|
||||
$ this-is-a-command
|
||||
This is output of the command
|
||||
|
||||
$ this-is-a-multiline-command with-some-arguments \
|
||||
and more arguments \
|
||||
and more arguments
|
||||
And some output.
|
||||
|
||||
"""
|
||||
|
||||
from pygments.lexer import RegexLexer, words
|
||||
from pygments.token import Comment, Generic, Keyword
|
||||
|
||||
_comment_style = Comment
|
||||
# Note: there is a slight inversion of styles to make it easier to read.
|
||||
# We highlight output with Prompt style and command as a normal text.
|
||||
_output_style = Generic.Prompt
|
||||
_command_style = Generic.Text
|
||||
_prompt_style = Keyword
|
||||
|
||||
|
||||
class CustomShellSessionLexer(RegexLexer):
|
||||
name = 'CustomShellSession'
|
||||
aliases = ['custom-shell-session']
|
||||
filenames = ['*.log']
|
||||
tokens = {
|
||||
'root': [
|
||||
(r'#.*\n', _comment_style),
|
||||
(r'^\$', _prompt_style, 'command'),
|
||||
(r'.', _output_style),
|
||||
],
|
||||
'command': [
|
||||
(r'\\\n', _command_style), # Continue in 'command' state.
|
||||
(r'$', _command_style, '#pop'), # End of line without escape.
|
||||
(r'.',
|
||||
_command_style), # Anything else continue in 'command' state.
|
||||
]
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
#
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
name='custom-shell-session',
|
||||
packages=find_packages(),
|
||||
entry_points="""
|
||||
[pygments.lexers]
|
||||
custom-shell-session = custom_shell_session.lexer:CustomShellSessionLexer
|
||||
""",
|
||||
)
|
||||
@@ -10,33 +10,37 @@ $ runtime/tools/wiki/build/build.py --deploy
|
||||
|
||||
# Markdown extensions
|
||||
|
||||
## Asides
|
||||
## Admonitions and Asides
|
||||
|
||||
Paragraphs wrapped into `<aside>...</aside>` will be rendered as a sidenote on
|
||||
margins of the page.
|
||||
Blockquotes starting with `> **Marker**` are converted either:
|
||||
|
||||
## Cross-references `@{ref|text}`
|
||||
- into sidenotes (if `Marker` is `Note`), which will be rendered on margins
|
||||
of the page;
|
||||
- admonitions (if `Marker` is `Source to read`, `Trying it` or `Warning`).
|
||||
|
||||
Cross-references are rendered as links to GitHub at the current commit.
|
||||
## Referencing C++ symbols and files
|
||||
|
||||
* `@{file-path}` is just rendered a link to the given file;
|
||||
* `@{package:name/path.dart}` is rendered as a link to file `path.dart` within
|
||||
package `name` - actual path is resolved via root `.packages` file in the SDK
|
||||
root;
|
||||
* `@{c++-symbol}` is rendered as a link to the line in the file which defines
|
||||
Script extends Markdown references with special support for references that
|
||||
use ``[`ref`][]`` and ``[text][`ref`]``. The following values for `ref` are
|
||||
recognized and resolved as links to GitHub at the current commit.
|
||||
|
||||
* `file-path` is resolved as a link to the given file;
|
||||
* `package:name/path.dart` is resolved as a link to file `path.dart` within
|
||||
package `name` - actual path is resolved via `.dart_tool/package_config.json`
|
||||
file in the SDK root;
|
||||
* `c++ symbol` is resolved as a link to the line in the file which defines
|
||||
the given C++ symbol.
|
||||
|
||||
If markdown file contains any references in this form then running
|
||||
`runtime/tools/wiki/build/build.py --deploy` will generate a reference
|
||||
section at the end of the file. Appending this section allows other Markdown
|
||||
tools (e.g. GitHub viewer) to render such special links correctly.
|
||||
|
||||
# Prerequisites
|
||||
|
||||
1. Install all Python dependencies.
|
||||
```console
|
||||
$ pip3 install coloredlogs jinja2 markdown aiohttp watchdog pymdown-extensions pygments
|
||||
```
|
||||
2. Install the custom pygments lexer we use for shell session examples:
|
||||
```
|
||||
$ cd runtime/tools/wiki/CustomShellSessionPygmentsLexer
|
||||
$ python3 setup.py develop
|
||||
```
|
||||
2. Install `libclang` (`brew install llvm` on Mac OS X).
|
||||
3. Install SASS compiler (make sure that SASS binary is in your path).
|
||||
4. Generate `xref.json` file following instructions in
|
||||
`xref_extractor/README.md`.
|
||||
@@ -0,0 +1,61 @@
|
||||
# Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
#
|
||||
"""Preprocesses Markdown source and converts admonitions written in blockquotes.
|
||||
|
||||
Usage: convert_admonitions(str)
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
_RECOGNIZED_ADMONITIONS = {
|
||||
"Source to read": "sourcecode",
|
||||
"Trying it": "tryit",
|
||||
"Warning": "warning"
|
||||
}
|
||||
|
||||
|
||||
def convert_admonitions(content: str) -> str:
|
||||
"""Convert blockquotes into admonitions if they start with a marker.
|
||||
|
||||
Blockquotes starting with `> **marker**` are converted either into
|
||||
sidenotes (`<span class="aside"/>`) or into admonitions to be
|
||||
processed by an admonition extension later.
|
||||
"""
|
||||
processed = []
|
||||
current_admonition = None
|
||||
indent = ''
|
||||
for line in content.split('\n'):
|
||||
if current_admonition is not None:
|
||||
if line.startswith('>'):
|
||||
processed.append(indent + line[1:])
|
||||
continue
|
||||
if current_admonition == 'Note':
|
||||
note = processed.pop()
|
||||
processed.pop()
|
||||
processed[-1] = processed[
|
||||
-1] + f' <span class="aside" markdown=1>{note}</span>'
|
||||
current_admonition = None
|
||||
elif line.startswith('> **') and line.endswith('**'):
|
||||
current_admonition = re.match(r'^> \*\*(.*)\*\*$', line)[1]
|
||||
if current_admonition == 'Note':
|
||||
indent = ''
|
||||
|
||||
# Drop all empy lines preceeding the side note.
|
||||
while processed[-1] == '':
|
||||
processed.pop()
|
||||
|
||||
# Do not try to attach sidenote to the section title.
|
||||
if processed[-1].startswith('#'):
|
||||
processed.append('')
|
||||
else:
|
||||
# Start an admonition using Python markdown syntax.
|
||||
processed.append(
|
||||
f'!!! {_RECOGNIZED_ADMONITIONS[current_admonition]} "{current_admonition}"'
|
||||
)
|
||||
current_admonition = True
|
||||
indent = ' '
|
||||
continue
|
||||
processed.append(line)
|
||||
return "\n".join(processed)
|
||||
@@ -17,35 +17,32 @@ If invoked with --deploy it would build deployment version in the
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, Sequence
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import codecs
|
||||
import coloredlogs
|
||||
import glob
|
||||
import jinja2
|
||||
import logging
|
||||
import markdown
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib
|
||||
|
||||
from aiohttp import web, WSCloseCode, WSMsgType
|
||||
from http.server import HTTPServer, SimpleHTTPRequestHandler
|
||||
from markdown.extensions.codehilite import CodeHiliteExtension
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, Sequence
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from watchdog.observers import Observer
|
||||
from xrefs import XrefExtension
|
||||
from admonitions import convert_admonitions
|
||||
|
||||
# Configure logging to use colors.
|
||||
coloredlogs.install(
|
||||
level='INFO', fmt='%(asctime)s - %(message)s', datefmt='%H:%M:%S')
|
||||
coloredlogs.install(level='INFO',
|
||||
fmt='%(asctime)s - %(message)s',
|
||||
datefmt='%H:%M:%S')
|
||||
|
||||
# Declare various directory paths.
|
||||
# We expected to be located in runtime/tools/wiki/build.
|
||||
@@ -73,16 +70,17 @@ os.makedirs(OUTPUT_CSS_DIR, exist_ok=True)
|
||||
# Parse incoming arguments.
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--deploy', dest='deploy', action='store_true')
|
||||
parser.add_argument('--deployment-root', dest='deployment_root', default='')
|
||||
parser.set_defaults(deploy=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
is_dev_mode = not args.deploy
|
||||
deployment_root = args.deployment_root
|
||||
|
||||
# Initialize jinja environment.
|
||||
jinja2_env = jinja2.Environment(
|
||||
loader=jinja2.FileSystemLoader(TEMPLATES_DIR),
|
||||
lstrip_blocks=True,
|
||||
trim_blocks=True)
|
||||
jinja2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATES_DIR),
|
||||
lstrip_blocks=True,
|
||||
trim_blocks=True)
|
||||
|
||||
|
||||
class Artifact:
|
||||
@@ -92,19 +90,19 @@ class Artifact:
|
||||
all: Dict[str, Artifact] = {}
|
||||
|
||||
# List of listeners which are notified whenever some artifact is rebuilt.
|
||||
listeners = []
|
||||
listeners: list[Callable] = []
|
||||
|
||||
def __init__(self, output: str, inputs: Sequence[str]):
|
||||
Artifact.all[output] = self
|
||||
self.output = output
|
||||
self.inputs = inputs
|
||||
self.inputs = [os.path.normpath(input) for input in inputs]
|
||||
|
||||
def depends_on(self, path: str) -> bool:
|
||||
"""Check if this"""
|
||||
return path in self.inputs
|
||||
|
||||
def build(self):
|
||||
pass
|
||||
"""Convert artifact inputs into an output."""
|
||||
|
||||
@staticmethod
|
||||
def build_all():
|
||||
@@ -112,11 +110,11 @@ class Artifact:
|
||||
Artifact.build_matching(lambda obj: True)
|
||||
|
||||
@staticmethod
|
||||
def build_matching(filter: Callable[[Artifact], bool]):
|
||||
def build_matching(predicate: Callable[[Artifact], bool]):
|
||||
"""Build all artifacts matching the given filter."""
|
||||
rebuilt = False
|
||||
for _, artifact in Artifact.all.items():
|
||||
if filter(artifact):
|
||||
if predicate(artifact):
|
||||
artifact.build()
|
||||
rebuilt = True
|
||||
|
||||
@@ -126,46 +124,74 @@ class Artifact:
|
||||
listener()
|
||||
|
||||
|
||||
xref_extension = XrefExtension()
|
||||
|
||||
|
||||
class Page(Artifact):
|
||||
"""A single wiki Page (a markdown file)."""
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
super().__init__(
|
||||
os.path.join(OUTPUT_DIR, name + '.html'),
|
||||
[os.path.join(WIKI_SOURCE_DIR, name + '.md')])
|
||||
super().__init__(os.path.join(OUTPUT_DIR, name + '.html'),
|
||||
[os.path.join(WIKI_SOURCE_DIR, name + '.md')])
|
||||
|
||||
def __repr__(self):
|
||||
return 'Page(%s <- %s)' % (self.output, self.inputs[0])
|
||||
return f'Page({self.output} <- {self.inputs[0]})'
|
||||
|
||||
def depends_on(self, path: str):
|
||||
return path.startswith(TEMPLATES_INCLUDES_DIR) or super().depends_on(
|
||||
path)
|
||||
|
||||
def load_markdown(self):
|
||||
with open(self.inputs[0], 'r') as file:
|
||||
def _load_markdown(self):
|
||||
with open(self.inputs[0], 'r', encoding='utf-8') as file:
|
||||
content = file.read()
|
||||
content = re.sub(r'(?<=[^\n])\n+<aside>', '<span class="aside">',
|
||||
content)
|
||||
content = re.sub(r'</aside>', '</span>', content)
|
||||
return content
|
||||
|
||||
# Remove autogenerated xref section.
|
||||
content = re.sub(r'<!\-\- AUTOGENERATED XREF SECTION \-\->.*$',
|
||||
'',
|
||||
content,
|
||||
flags=re.DOTALL)
|
||||
|
||||
return convert_admonitions(content)
|
||||
|
||||
def _update_xref_section(self, xrefs):
|
||||
with open(self.inputs[0], 'r', encoding='utf-8') as file:
|
||||
content = file.read()
|
||||
section = '\n'.join(
|
||||
['', '<!-- AUTOGENERATED XREF SECTION -->'] +
|
||||
[f'[{key}]: {value}' for key, value in xrefs.items()])
|
||||
with open(self.inputs[0], 'w', encoding='utf-8') as file:
|
||||
content = re.sub(r'\n<!-- AUTOGENERATED XREF SECTION -->.*$',
|
||||
'',
|
||||
content,
|
||||
flags=re.DOTALL)
|
||||
content += section
|
||||
file.write(content)
|
||||
|
||||
def build(self):
|
||||
logging.info('Build %s from %s', self.output, self.inputs[0])
|
||||
|
||||
template = jinja2_env.get_template(PAGE_TEMPLATE)
|
||||
md_converter = markdown.Markdown(extensions=[
|
||||
'admonition',
|
||||
'extra',
|
||||
CodeHiliteExtension(),
|
||||
'tables',
|
||||
'pymdownx.superfences',
|
||||
'toc',
|
||||
xref_extension,
|
||||
])
|
||||
result = template.render({
|
||||
'dev':
|
||||
is_dev_mode,
|
||||
'body':
|
||||
markdown.markdown(
|
||||
self.load_markdown(),
|
||||
extensions=[
|
||||
'admonition', 'extra',
|
||||
CodeHiliteExtension(), 'tables', 'pymdownx.superfences',
|
||||
XrefExtension()
|
||||
])
|
||||
md_converter.convert(self._load_markdown()),
|
||||
'root':
|
||||
deployment_root
|
||||
})
|
||||
# pylint: disable=no-member
|
||||
if not is_dev_mode and len(md_converter.xrefs) > 0:
|
||||
self._update_xref_section(md_converter.xrefs)
|
||||
|
||||
os.makedirs(os.path.dirname(self.output), exist_ok=True)
|
||||
with codecs.open(self.output, "w", encoding='utf-8') as file:
|
||||
@@ -180,12 +206,11 @@ class Style(Artifact):
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
super().__init__(
|
||||
os.path.join(OUTPUT_CSS_DIR, name + '.css'),
|
||||
[os.path.join(STYLES_DIR, name + '.scss')])
|
||||
super().__init__(os.path.join(OUTPUT_CSS_DIR, name + '.css'),
|
||||
[os.path.join(STYLES_DIR, name + '.scss')])
|
||||
|
||||
def __repr__(self):
|
||||
return 'Style(%s <- %s)' % (self.output, self.inputs[0])
|
||||
return f'Style({self.output} <- {self.inputs[0]})'
|
||||
|
||||
def depends_on(self, path: str):
|
||||
return path.startswith(STYLES_INCLUDES_DIR) or super().depends_on(path)
|
||||
@@ -206,15 +231,17 @@ def find_images_directories():
|
||||
def find_artifacts():
|
||||
"""Find all wiki pages and styles and create corresponding Artifacts."""
|
||||
Artifact.all = {}
|
||||
for f in Path(WIKI_SOURCE_DIR).rglob('*.md'):
|
||||
name = f.relative_to(Path(WIKI_SOURCE_DIR)).as_posix().rsplit('.', 1)[0]
|
||||
for file in Path(WIKI_SOURCE_DIR).rglob('*.md'):
|
||||
name = file.relative_to(Path(WIKI_SOURCE_DIR)).as_posix().rsplit(
|
||||
'.', 1)[0]
|
||||
Page(name)
|
||||
|
||||
for f in Path(STYLES_DIR).glob('*.scss'):
|
||||
Style(f.stem)
|
||||
for file in Path(STYLES_DIR).glob('*.scss'):
|
||||
Style(file.stem)
|
||||
|
||||
|
||||
def build_for_deploy():
|
||||
"""Create a directory which can be deployed to static hosting."""
|
||||
logging.info('Building wiki for deployment into %s', OUTPUT_DIR)
|
||||
Artifact.build_all()
|
||||
for images_dir in find_images_directories():
|
||||
@@ -234,15 +261,13 @@ def build_for_deploy():
|
||||
class ArtifactEventHandler(FileSystemEventHandler):
|
||||
"""File system listener rebuilding artifacts based on changed paths."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def on_modified(self, event):
|
||||
Artifact.build_matching(
|
||||
lambda artifact: artifact.depends_on(event.src_path))
|
||||
path = os.path.relpath(event.src_path, '.')
|
||||
Artifact.build_matching(lambda artifact: artifact.depends_on(path))
|
||||
|
||||
|
||||
def serve_for_development():
|
||||
"""Serve wiki for development (with hot refresh)."""
|
||||
logging.info('Serving wiki for development')
|
||||
Artifact.build_all()
|
||||
|
||||
@@ -257,8 +282,8 @@ def serve_for_development():
|
||||
|
||||
async def on_shutdown(app):
|
||||
for ws in app['websockets']:
|
||||
await ws.close(
|
||||
code=WSCloseCode.GOING_AWAY, message='Server shutdown')
|
||||
await ws.close(code=WSCloseCode.GOING_AWAY,
|
||||
message='Server shutdown')
|
||||
observer.stop()
|
||||
observer.join()
|
||||
|
||||
@@ -320,6 +345,7 @@ def serve_for_development():
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
find_artifacts()
|
||||
if is_dev_mode:
|
||||
serve_for_development()
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
# Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
#
|
||||
"""Clang based C++ code indexer which produces xref.json."""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Optional, Set
|
||||
|
||||
from marshmallow_dataclass import class_schema
|
||||
from progress.bar import ShadyBar
|
||||
|
||||
_libclang_path = None
|
||||
_clang_include_dir = None
|
||||
if platform.system() == 'Darwin':
|
||||
_path_to_llvm = subprocess.check_output(['brew', '--prefix', 'llvm'],
|
||||
encoding='utf-8').strip()
|
||||
_clang_include_dir = glob.glob(f'{_path_to_llvm}/lib/clang/**/include')[0]
|
||||
_libclang_path = f'{_path_to_llvm}/lib/libclang.dylib'
|
||||
sys.path.append(
|
||||
f'{_path_to_llvm}/lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages'
|
||||
)
|
||||
|
||||
# pylint: disable=wrong-import-position,line-too-long
|
||||
from clang.cindex import Config, CompilationDatabase, Cursor, CursorKind, Index, SourceLocation, TranslationUnit
|
||||
|
||||
if _libclang_path is not None:
|
||||
Config.set_library_file(_libclang_path)
|
||||
|
||||
_DART_CONFIGURATION = 'Release' + (
|
||||
'ARM64' if platform.uname().machine == 'arm64' else 'X64')
|
||||
_DART_BUILD_DIR = ('xcodebuild' if platform.system() == 'Darwin' else
|
||||
'out') + '/' + _DART_CONFIGURATION
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _change_working_directory_to(path):
|
||||
oldpwd = os.getcwd()
|
||||
os.chdir(path)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
os.chdir(oldpwd)
|
||||
|
||||
|
||||
def _create_compilation_database():
|
||||
"""Create compilation database for the default configuration.
|
||||
|
||||
Extacts compilation database (compile_commands.json) for the default
|
||||
build configuration and filters it down to commands taht builds just
|
||||
the core VM pieces in the host configuration.
|
||||
"""
|
||||
logging.info('Extracting compilation commands from build files for %s',
|
||||
_DART_CONFIGURATION)
|
||||
with _change_working_directory_to(_DART_BUILD_DIR):
|
||||
commands = json.loads(
|
||||
subprocess.check_output(['ninja', '-C', '.', '-t', 'compdb',
|
||||
'cxx']))
|
||||
pattern = re.compile(
|
||||
r'libdart(_vm|_compiler)?_precompiler_host_targeting_host\.')
|
||||
with open('compile_commands.json', 'w', encoding='utf-8') as outfile:
|
||||
json.dump([
|
||||
cmd for cmd in commands
|
||||
if pattern.search(cmd['command']) is not None
|
||||
], outfile)
|
||||
|
||||
|
||||
def _get_current_commit_hash():
|
||||
return subprocess.check_output(['git', 'merge-base', 'main', 'HEAD'],
|
||||
text=True).strip()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ClassInfo:
|
||||
location: str
|
||||
members: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Location:
|
||||
"""Symbol location referring to a line in a specific file."""
|
||||
filename: str
|
||||
lineno: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class SymbolsIndex:
|
||||
"""Index of C++ symbols extracted from source."""
|
||||
commit: str
|
||||
files: list[str] = field(default_factory=list)
|
||||
classes: Dict[str, _ClassInfo] = field(default_factory=dict)
|
||||
functions: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def try_resolve(self, ref: str) -> Optional[Location]:
|
||||
"""Resolve the location of the given reference."""
|
||||
loc = self._try_resolve_impl(ref)
|
||||
if loc is not None:
|
||||
return self._location_from_string(loc)
|
||||
return None
|
||||
|
||||
def _try_resolve_impl(self, ref: str) -> Optional[str]:
|
||||
if ref in self.functions:
|
||||
return self.functions[ref]
|
||||
if ref in self.classes:
|
||||
return self.classes[ref].location
|
||||
if '::' in ref:
|
||||
(class_name, function_name) = ref.rsplit('::', 1)
|
||||
if class_name in self.classes:
|
||||
return self.classes[class_name].members.get(function_name)
|
||||
return None
|
||||
|
||||
def _location_from_string(self, loc: str) -> Location:
|
||||
(file_idx, line_idx) = loc.split(':', 1)
|
||||
return Location(self.files[int(file_idx)], int(line_idx))
|
||||
|
||||
|
||||
class _Indexer:
|
||||
symbols_index: SymbolsIndex
|
||||
|
||||
processed_files: Set[str]
|
||||
classes_by_usr: Dict[str, _ClassInfo]
|
||||
|
||||
name_stack: list[str]
|
||||
info_stack: list[_ClassInfo]
|
||||
files_seen_in_unit: Set[str]
|
||||
|
||||
def __init__(self):
|
||||
self.symbols_index = SymbolsIndex(commit=_get_current_commit_hash())
|
||||
|
||||
self.processed_files = set()
|
||||
self.classes_by_usr = {}
|
||||
self.files_index = {}
|
||||
|
||||
self.name_stack = []
|
||||
self.info_stack = []
|
||||
self.files_seen_in_unit = set()
|
||||
|
||||
def index(self, unit: TranslationUnit):
|
||||
"""Index the given translation unit and append new symbols to index."""
|
||||
self.name_stack.clear()
|
||||
self.info_stack.clear()
|
||||
self.files_seen_in_unit.clear()
|
||||
self._recurse(unit.cursor)
|
||||
self.processed_files |= self.files_seen_in_unit
|
||||
|
||||
def _recurse(self, cursor: Cursor):
|
||||
name = ""
|
||||
kind = cursor.kind
|
||||
|
||||
if cursor.location.file is not None:
|
||||
name = cursor.location.file.name
|
||||
if name in self.processed_files or not name.startswith('../..'):
|
||||
return
|
||||
self.files_seen_in_unit.add(name)
|
||||
|
||||
if kind == CursorKind.CLASS_DECL:
|
||||
if not cursor.is_definition():
|
||||
return
|
||||
usr = cursor.get_usr()
|
||||
if usr in self.classes_by_usr:
|
||||
return
|
||||
self.name_stack.append(cursor.spelling)
|
||||
class_name = '::'.join(self.name_stack)
|
||||
class_info = _ClassInfo(self._format_location(cursor.location))
|
||||
self.info_stack.append(class_info)
|
||||
self.symbols_index.classes[class_name] = class_info
|
||||
self.classes_by_usr[usr] = class_info
|
||||
elif kind == CursorKind.NAMESPACE:
|
||||
self.name_stack.append(cursor.spelling)
|
||||
elif kind == CursorKind.FUNCTION_DECL and cursor.is_definition():
|
||||
namespace_prefix = ""
|
||||
if cursor.semantic_parent.kind == CursorKind.NAMESPACE:
|
||||
namespace_prefix = '::'.join(self.name_stack) + (
|
||||
'::' if len(self.name_stack) > 0 else "")
|
||||
function_name = namespace_prefix + cursor.spelling
|
||||
self.symbols_index.functions[function_name] = self._format_location(
|
||||
cursor.location)
|
||||
return
|
||||
elif kind == CursorKind.CXX_METHOD and cursor.is_definition():
|
||||
parent = cursor.semantic_parent
|
||||
if parent.kind == CursorKind.CLASS_DECL:
|
||||
class_info_or_none = self.classes_by_usr.get(parent.get_usr())
|
||||
if class_info_or_none is None:
|
||||
return
|
||||
class_info_or_none.members[
|
||||
cursor.spelling] = self._format_location(cursor.location)
|
||||
return
|
||||
elif kind == CursorKind.VAR_DECL and cursor.is_definition():
|
||||
parent = cursor.semantic_parent
|
||||
if parent.kind == CursorKind.CLASS_DECL:
|
||||
class_info_or_none = self.classes_by_usr.get(parent.get_usr())
|
||||
if class_info_or_none is None:
|
||||
return
|
||||
class_info_or_none.members[
|
||||
cursor.spelling] = self._format_location(cursor.location)
|
||||
|
||||
for child in cursor.get_children():
|
||||
self._recurse(child)
|
||||
|
||||
if kind == CursorKind.NAMESPACE:
|
||||
self.name_stack.pop()
|
||||
elif kind == CursorKind.CLASS_DECL:
|
||||
self.name_stack.pop()
|
||||
self.info_stack.pop()
|
||||
|
||||
def _format_location(self, loc: SourceLocation):
|
||||
file_name = loc.file.name
|
||||
lineno = loc.line
|
||||
return f'{self._get_file_index(file_name)}:{lineno}'
|
||||
|
||||
def _get_file_index(self, file_name: str):
|
||||
index = self.files_index.get(file_name)
|
||||
if index is None:
|
||||
index = len(self.symbols_index.files)
|
||||
self.files_index[file_name] = index
|
||||
self.symbols_index.files.append(
|
||||
os.path.relpath(os.path.abspath(file_name),
|
||||
os.path.abspath('../..')))
|
||||
return index
|
||||
|
||||
|
||||
def _index_source() -> SymbolsIndex:
|
||||
indexer = _Indexer()
|
||||
|
||||
_create_compilation_database()
|
||||
with _change_working_directory_to(_DART_BUILD_DIR):
|
||||
index = Index.create()
|
||||
compdb = CompilationDatabase.fromDirectory('.')
|
||||
|
||||
commands = list(compdb.getAllCompileCommands())
|
||||
with ShadyBar('Indexing',
|
||||
max=len(commands),
|
||||
suffix='%(percent)d%% eta %(eta_td)s') as progress_bar:
|
||||
for command in commands:
|
||||
args = [
|
||||
arg for arg in command.arguments
|
||||
if arg.startswith('-I') or arg.startswith('-W') or
|
||||
arg.startswith('-D') or arg.startswith('-i') or
|
||||
arg.startswith('sdk/') or arg.startswith('-std')
|
||||
] + [
|
||||
'-Wno-macro-redefined', '-Wno-unused-const-variable',
|
||||
'-Wno-unused-function', '-Wno-unused-variable'
|
||||
]
|
||||
|
||||
if _clang_include_dir is not None:
|
||||
args.append(f'-I{_clang_include_dir}')
|
||||
|
||||
unit = index.parse(command.filename, args=args)
|
||||
for diag in unit.diagnostics:
|
||||
print(diag.format())
|
||||
|
||||
indexer.index(unit)
|
||||
progress_bar.next()
|
||||
|
||||
return indexer.symbols_index
|
||||
|
||||
|
||||
_SymbolsIndexSchema = class_schema(SymbolsIndex)()
|
||||
|
||||
|
||||
def load_index(filename: str) -> SymbolsIndex:
|
||||
"""Load symbols index from the given file.
|
||||
|
||||
If index is out of date or missing it will be generated.
|
||||
"""
|
||||
index: SymbolsIndex
|
||||
|
||||
if os.path.exists(filename):
|
||||
with open(filename, 'r', encoding='utf-8') as json_file:
|
||||
index = _SymbolsIndexSchema.loads(json_file.read())
|
||||
if _get_current_commit_hash() == index.commit:
|
||||
logging.info('Loaded symbols index from %s', filename)
|
||||
return index
|
||||
logging.warning(
|
||||
'%s is generated for commit %s while current commit is %s',
|
||||
filename, index.commit, _get_current_commit_hash())
|
||||
|
||||
index = _index_source()
|
||||
with open(filename, 'w', encoding='utf-8') as json_file:
|
||||
json_file.write(_SymbolsIndexSchema.dumps(index))
|
||||
logging.info(
|
||||
'Successfully indexed C++ source and written symbols index into %s',
|
||||
filename)
|
||||
return index
|
||||
@@ -2,119 +2,158 @@
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
#
|
||||
"""Support for @{ref|text} reference links in Markdown.
|
||||
"""Markdown extension for xrefs and reference to other Markdown files.
|
||||
|
||||
This Markdown extension converts @{ref|text} into a link with the given
|
||||
[text] pointing to a particular source code location. [ref] can be one of
|
||||
the following:
|
||||
Xref is a reference of form [`symbol`][] or [text][`symbol`], where symbol
|
||||
is expected to be one of the following:
|
||||
|
||||
* package:-scheme URI - it will be resolved using .packages file in the
|
||||
root directory
|
||||
* file path
|
||||
* C++ symbol - will be resolved through xref.json file (see README.md)
|
||||
|
||||
Xrefs are converted to GitHub links.
|
||||
|
||||
Additionally this extension retargets links pointing to markdown files to
|
||||
the html files produced from these markdown files.
|
||||
|
||||
Usage: markdown.markdown(extensions=[XrefExtension()])
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
|
||||
from markdown.extensions import Extension
|
||||
from markdown.inlinepatterns import InlineProcessor
|
||||
from markdown.util import etree
|
||||
from typing import Optional
|
||||
import xml.etree.ElementTree as etree
|
||||
from typing import Dict, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
_current_commit_hash = subprocess.run(['git', 'rev-parse', 'HEAD'],
|
||||
capture_output=True,
|
||||
encoding='utf8').stdout
|
||||
|
||||
# Load .packages file into a dictionary.
|
||||
with open('.packages') as packages_file:
|
||||
_packages = dict([
|
||||
package_mapping.split(':', 1)
|
||||
for package_mapping in packages_file
|
||||
if not package_mapping.startswith('#')
|
||||
])
|
||||
|
||||
# Load xref.json and verify that it was generated for the current commit to
|
||||
# avoid discrepancies in the generated links.
|
||||
with open('xref.json') as json_file:
|
||||
_xrefs = json.load(json_file)
|
||||
|
||||
if _current_commit_hash != _xrefs['commit']:
|
||||
logging.error(
|
||||
'xref.json is generated for commit %s while current commit is %s',
|
||||
_xrefs['commit'], _current_commit_hash)
|
||||
|
||||
|
||||
def _make_github_uri(file: str, lineno: str = None) -> str:
|
||||
"""Generates source link pointing to GitHub"""
|
||||
fragment = '#L%s' % (lineno) if lineno is not None else ''
|
||||
return 'https://github.com/dart-lang/sdk/blob/%s/%s%s' % (
|
||||
_current_commit_hash, file, fragment)
|
||||
|
||||
|
||||
def _file_ref_to_github_uri(file_ref: str) -> str:
|
||||
"""Generates source link pointing to GitHub from an xref.json reference."""
|
||||
(file_idx, line_idx) = file_ref.split(':', 1)
|
||||
return _make_github_uri(_xrefs['files'][int(file_idx)], line_idx)
|
||||
|
||||
|
||||
def _resolve_ref_via_xref(ref: str) -> Optional[str]:
|
||||
"""Resolve the target of the given reference via xref.json"""
|
||||
if ref in _xrefs['functions']:
|
||||
return _xrefs['functions'][ref]
|
||||
if ref in _xrefs['classes']:
|
||||
return _xrefs['classes'][ref][0]
|
||||
if '::' in ref:
|
||||
(class_name, function_name) = ref.rsplit('::', 1)
|
||||
if class_name in _xrefs['classes'] and len(
|
||||
_xrefs['classes'][class_name]) == 2:
|
||||
return _xrefs['classes'][class_name][1][function_name]
|
||||
logging.error('Failed to resolve xref %s' % ref)
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_ref(ref: str) -> Optional[str]:
|
||||
if ref.startswith('package:'):
|
||||
# Resolve as package uri via .packages.
|
||||
uri = urlparse(ref)
|
||||
(package_name, *path_to_file) = uri.path.split('/', 1)
|
||||
package_path = _packages[package_name]
|
||||
if len(path_to_file) == 0:
|
||||
return _make_github_uri(package_path)
|
||||
else:
|
||||
return _make_github_uri(os.path.join(package_path, path_to_file[0]))
|
||||
elif os.path.exists(ref):
|
||||
# Resolve as a file link.
|
||||
return _make_github_uri(_current_commit_hash, ref)
|
||||
else:
|
||||
# Resolve as a C++ symbol via xref.json
|
||||
file_ref = _resolve_ref_via_xref(ref)
|
||||
if file_ref is not None:
|
||||
return _file_ref_to_github_uri(file_ref)
|
||||
from cpp_indexer import SymbolsIndex, load_index
|
||||
from markdown.extensions import Extension
|
||||
from markdown.inlinepatterns import InlineProcessor
|
||||
from markdown.treeprocessors import Treeprocessor
|
||||
|
||||
|
||||
class _XrefPattern(InlineProcessor):
|
||||
"""InlineProcessor responsible for handling @{ref|text} syntax."""
|
||||
"""Converts xrefs into GitHub links.
|
||||
|
||||
Recognizes [`symbol`][] and [text][`symbol`] link formats where symbol
|
||||
is expected to be one of the following:
|
||||
|
||||
* Fully qualified reference to a C++ class, method or function;
|
||||
* Package URI pointing to one of the packages included in the SDK
|
||||
checkout.
|
||||
* File reference to one of the file in the SDK.
|
||||
"""
|
||||
|
||||
XREF_RE = r'\[`(?P<symbol>[^]]+)`\]?\[\]|\[(?P<text>[^]]*)\]\[`(?P<target>[^]]+)`\]'
|
||||
|
||||
def __init__(self, md, symbols_index: SymbolsIndex,
|
||||
packages: Dict[str, str]):
|
||||
super().__init__(_XrefPattern.XREF_RE)
|
||||
self.symbols_index = symbols_index
|
||||
self.packages = packages
|
||||
self.md = md
|
||||
|
||||
def handleMatch(self, m, data):
|
||||
ref = m.group(1)
|
||||
text = m.group(2)
|
||||
uri = _resolve_ref(ref)
|
||||
el = etree.Element('a')
|
||||
el.attrib['href'] = uri
|
||||
el.attrib['target'] = 'blank'
|
||||
el.text = text[1:] if text is not None else ref
|
||||
return el, m.start(0), m.end(0)
|
||||
text = m.group('text')
|
||||
symbol = m.group('symbol')
|
||||
if symbol is None:
|
||||
symbol = m.group('target')
|
||||
|
||||
uri = self._resolve_ref(symbol) or '#broken-link'
|
||||
|
||||
# Remember this xref. build process can later use this information
|
||||
# to produce xref reference section at the end of the markdown file.
|
||||
self.md.xrefs[f"`{symbol}`"] = uri
|
||||
|
||||
# Create <a href='uri'>text</a> element. If text is not defined
|
||||
# simply use a slightly sanitized symbol name.
|
||||
anchor = etree.Element('a')
|
||||
anchor.attrib['href'] = uri
|
||||
anchor.attrib['target'] = '_blank'
|
||||
if text is not None:
|
||||
anchor.text = text
|
||||
else:
|
||||
code = etree.Element('code')
|
||||
code.text = re.sub(r'^dart::', '', symbol)
|
||||
anchor.append(code)
|
||||
|
||||
# Replace the whole pattern match with anchor element.
|
||||
return anchor, m.start(0), m.end(0)
|
||||
|
||||
def _resolve_ref(self, ref: str) -> Optional[str]:
|
||||
if ref.startswith('package:'):
|
||||
# Resolve as package uri via .packages.
|
||||
uri = urlparse(ref)
|
||||
(package_name, *path_to_file) = uri.path.split('/', 1)
|
||||
package_path = self.packages[package_name]
|
||||
if len(path_to_file) == 0:
|
||||
return self._make_github_uri(package_path)
|
||||
else:
|
||||
return self._make_github_uri(
|
||||
os.path.join(package_path, path_to_file[0]))
|
||||
elif os.path.exists(ref):
|
||||
# Resolve as a file link.
|
||||
return self._make_github_uri(ref)
|
||||
else:
|
||||
# Resolve as a symbol.
|
||||
loc = self.symbols_index.try_resolve(ref)
|
||||
if loc is not None:
|
||||
return self._make_github_uri(loc.filename, loc.lineno)
|
||||
|
||||
logging.error('Failed to resolve xref %s', ref)
|
||||
return None
|
||||
|
||||
def _make_github_uri(self, file: str, lineno: Optional[int] = None) -> str:
|
||||
"""Generates source link pointing to GitHub"""
|
||||
fragment = f'#L{lineno}' if lineno is not None else ''
|
||||
return f'https://github.com/dart-lang/sdk/blob/{self.symbols_index.commit}/{file}{fragment}'
|
||||
|
||||
|
||||
class _MdLinkFixerTreeprocessor(Treeprocessor):
|
||||
"""Redirects links pointing to .md files to .html files built from them."""
|
||||
|
||||
def run(self, root):
|
||||
for elem in root.iter('a'):
|
||||
href = elem.get('href')
|
||||
if href is None:
|
||||
continue
|
||||
parsed_href = urlparse(href)
|
||||
if parsed_href.path.endswith('.md'):
|
||||
elem.set(
|
||||
'href',
|
||||
parsed_href._replace(path=parsed_href.path[:-3] +
|
||||
'.html').geturl())
|
||||
|
||||
|
||||
class XrefExtension(Extension):
|
||||
"""Markdown extension responsible for expanding @{ref|text} into links."""
|
||||
"""Markdown extension which handles xrefs and links to markdown files."""
|
||||
symbols_index: SymbolsIndex
|
||||
packages: Dict[str, str]
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.symbols_index = load_index('xref.json')
|
||||
self.packages = XrefExtension._load_package_config()
|
||||
|
||||
def extendMarkdown(self, md):
|
||||
md.xrefs = {}
|
||||
md.treeprocessors.register(_MdLinkFixerTreeprocessor(), 'mdlinkfixer',
|
||||
0)
|
||||
md.inlinePatterns.register(
|
||||
_XrefPattern(r'@{([^}|]*)(\|[^}]+)?}'), 'xref', 175)
|
||||
_XrefPattern(md, self.symbols_index, self.packages), 'xref', 200)
|
||||
|
||||
@staticmethod
|
||||
def _load_package_config() -> Dict[str, str]:
|
||||
# Load package_config.json file into a dictionary.
|
||||
with open('.dart_tool/package_config.json',
|
||||
encoding='utf-8') as package_config_file:
|
||||
package_config = json.load(package_config_file)
|
||||
return dict([(pkg['name'],
|
||||
os.path.normpath(
|
||||
os.path.join('.dart_tool/', pkg['rootUri'],
|
||||
pkg['packageUri'])))
|
||||
for pkg in package_config['packages']
|
||||
if 'packageUri' in pkg])
|
||||
|
||||
@@ -1,7 +1,26 @@
|
||||
@import url('https://fonts.googleapis.com/css?family=Source+Code+Pro|Source+Sans+Pro&display=swap');
|
||||
@import url('https://rsms.me/inter/inter.css');
|
||||
@import url('https://rsms.me/inter/inter-display.css');
|
||||
|
||||
$monospace: 'Source Code Pro', Menlo, Consolas, monospace;
|
||||
$sans: 'Source Sans Pro', sans-serif;
|
||||
@font-face {
|
||||
font-named-instance:"Regular";font-family: jbmono;
|
||||
font-style: normal;
|
||||
font-weight: 100 800;
|
||||
src: url(https://rsms.me/res/fonts/jbm/jetbrains-mono-wght.woff2) format("woff2")
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-named-instance:"Italic";font-family: jbmono;
|
||||
font-style: italic;
|
||||
font-weight: 100 800;
|
||||
src: url(https://rsms.me/res/fonts/jbm/jetbrains-mono-italic_wght.woff2) format("woff2")
|
||||
}
|
||||
|
||||
:root { font-family: 'Inter', sans-serif; }
|
||||
@supports (font-variation-settings: normal) {
|
||||
:root { font-family: 'Inter var', sans-serif; font-weight: 460;}
|
||||
}
|
||||
|
||||
$monospace: jbmono, 'IBM Plex Mono', Menlo, Consolas, monospace;
|
||||
|
||||
a {
|
||||
color: #C00;
|
||||
@@ -27,7 +46,7 @@ $code-font-size: 16px;
|
||||
$code-line-height: 24px;
|
||||
$header-font-size: 16px;
|
||||
|
||||
div.codehilite {
|
||||
div.highlight {
|
||||
font-size: $header-font-size;
|
||||
margin: 0px -20px;
|
||||
padding: 5px 20px;
|
||||
@@ -48,8 +67,9 @@ html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
height: 100%;
|
||||
font: normal #{$base-font-size}/#{$base-line-height} $sans;
|
||||
text-align: justify;
|
||||
font-size: $base-font-size;
|
||||
line-height: $base-line-height;
|
||||
text-align: left;
|
||||
color: #112;
|
||||
text-rendering: optimizeLegibility;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
@@ -57,7 +77,6 @@ html, body {
|
||||
}
|
||||
|
||||
h1, h2, h3, h4, h5 {
|
||||
font-family: $sans;
|
||||
text-align: left;
|
||||
position: relative;
|
||||
}
|
||||
@@ -151,45 +170,28 @@ p > img {
|
||||
position: relative;
|
||||
right: 0px;
|
||||
width: auto;
|
||||
text-align: justify;
|
||||
font-size: .9em;
|
||||
background: #F0F4C3;
|
||||
text-align: left;
|
||||
font-size: .8em;
|
||||
line-height: 1.5em;
|
||||
background-color: rgba(128,128,128,0.1);
|
||||
display: block;
|
||||
padding: 5px;
|
||||
padding: .8em;
|
||||
}
|
||||
|
||||
body {
|
||||
font: normal 14px/18px $sans;
|
||||
font-size: 14px;
|
||||
line-height: 18px;
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
div.codehilite {
|
||||
font: normal 10px/13px $monospace;
|
||||
div.highlight {
|
||||
font-size: 10px;
|
||||
line-height: 13px;
|
||||
margin: 0px -5px;
|
||||
padding: 5px 5px;
|
||||
}
|
||||
}
|
||||
|
||||
.book-title {
|
||||
font: 3em $monospace;
|
||||
text-transform: uppercase;
|
||||
font-weight: normal;
|
||||
/* border-width: 0px 1px 0px 1px; */
|
||||
/* border: dashed #C00; */
|
||||
border-width: 2px 0px 2px;
|
||||
padding: 10px 0px 5px 0px;
|
||||
text-align: center;
|
||||
letter-spacing: .1em;
|
||||
color: #C00;
|
||||
/* background-color: white; */
|
||||
}
|
||||
|
||||
.book-subtitle {
|
||||
font-family: $monospace;
|
||||
text-align: center;
|
||||
letter-spacing: .1em;
|
||||
}
|
||||
|
||||
code {
|
||||
font: normal #{$code-font-size}/#{$code-line-height} $monospace;
|
||||
white-space: pre;
|
||||
@@ -211,7 +213,6 @@ pre {
|
||||
}
|
||||
|
||||
.start-reading span {
|
||||
font-family: $sans;
|
||||
color: white;
|
||||
background-color: #C00;
|
||||
border-radius: 10px;
|
||||
@@ -224,82 +225,125 @@ pre {
|
||||
}
|
||||
|
||||
// Pygments
|
||||
.codehilite {
|
||||
background-color: #fdf6e3; color: #586e75;
|
||||
.hll {
|
||||
opacity: .4;
|
||||
.highlight {
|
||||
line-height: initial;
|
||||
|
||||
pre {
|
||||
--bg: #fff;
|
||||
--fg: #000;
|
||||
--comment: #6a737d;
|
||||
--punctuation: inherit;
|
||||
--operator: inherit;
|
||||
--entity: #6f42c1;
|
||||
--keyword: #d73a49;
|
||||
--variable: #e36209;
|
||||
--string-constant: #032f62; /* strings etc */
|
||||
--numeric-constant: #005cc5;
|
||||
|
||||
padding: 1em;
|
||||
background: var(--bg);
|
||||
color: var(--fg);
|
||||
|
||||
/* punctuation */
|
||||
& .p { color: var(--punctuation); }
|
||||
|
||||
/* syntax error */
|
||||
/*& .err { color: #ff0015; }*/
|
||||
|
||||
/* names / identifiers */
|
||||
/*& .n ,*/ /* Name */
|
||||
& .na, /* Name.Attribute */
|
||||
& .nb, /* Name.Builtin */
|
||||
& .nc, /* Name.Class */
|
||||
& .no, /* Name.Constant */
|
||||
& .nd, /* Name.Decorator */
|
||||
& .ni, /* Name.Entity */
|
||||
& .ne, /* Name.Exception */
|
||||
& .nf, /* Name.Function */
|
||||
& .nl, /* Name.Label */
|
||||
& .nn, /* Name.Namespace */
|
||||
& .nx,
|
||||
& .py,
|
||||
& .nt /* Name.Tag */ {
|
||||
color: var(--entity);
|
||||
}
|
||||
|
||||
& .vc,
|
||||
& .vg,
|
||||
& .vi,
|
||||
& .nv {
|
||||
color: var(--variable);
|
||||
}
|
||||
|
||||
& .bp { /* Builtin.Pseudo */
|
||||
|
||||
}
|
||||
|
||||
& .o,
|
||||
& .ow { color: var(--operator); }
|
||||
|
||||
& .c,
|
||||
& .cm,
|
||||
& .cp,
|
||||
& .c1,
|
||||
& .cs { color: var(--comment); font-style: italic; }
|
||||
|
||||
/* Keywords */
|
||||
& .k,
|
||||
& .kc,
|
||||
& .kd,
|
||||
& .kn,
|
||||
& .kp,
|
||||
& .kr,
|
||||
& .kt { color: var(--keyword); font-weight: 500; }
|
||||
|
||||
/* strings */
|
||||
& .s,
|
||||
& .sb,
|
||||
& .sd,
|
||||
& .sc,
|
||||
& .s2,
|
||||
& .se,
|
||||
& .sh,
|
||||
& .si,
|
||||
& .sx,
|
||||
& .sr,
|
||||
& .ss,
|
||||
& .s1 { color: var(--string-constant); }
|
||||
|
||||
/* number */
|
||||
& .m, & .mi, & .mf { color: var(--numeric-constant); }
|
||||
|
||||
& .gi /* Generic.Inserted */ {
|
||||
color: #22863a;
|
||||
background-color: #f0fff4;
|
||||
}
|
||||
|
||||
& .gd /* Generic.Deleted */ {
|
||||
color: #b31d28;
|
||||
background-color: #ffeef0;
|
||||
}
|
||||
|
||||
& .gp /* Generic.Prompt */ {
|
||||
color: var(--keyword);
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
& .go /* Generic.Output */ {
|
||||
color: #003b7e;
|
||||
}
|
||||
|
||||
/* ? */
|
||||
/*& .l,
|
||||
& .ld,
|
||||
& .m,
|
||||
& .mf,
|
||||
& .mh,
|
||||
& .mi,
|
||||
& .mo,
|
||||
& .nx,
|
||||
& .il { color: #ff0000; }*/
|
||||
}
|
||||
.c { color: #93a1a1 } /* Comment */
|
||||
.err { color: #586e75 } /* Error */
|
||||
.g { color: #586e75 } /* Generic */
|
||||
.k { color: #859900 } /* Keyword */
|
||||
.l { color: #586e75 } /* Literal */
|
||||
.n { color: #586e75 } /* Name */
|
||||
.o { color: #859900 } /* Operator */
|
||||
.x { color: #cb4b16 } /* Other */
|
||||
.p { color: #586e75 } /* Punctuation */
|
||||
.cm { color: #93a1a1 } /* Comment.Multiline */
|
||||
.cp { color: #859900 } /* Comment.Preproc */
|
||||
.c1 { color: #93a1a1 } /* Comment.Single */
|
||||
.cs { color: #859900 } /* Comment.Special */
|
||||
.c, .cm, .cp, .c1, .cs {
|
||||
font-style: italic;
|
||||
}
|
||||
.gd { color: #2aa198 } /* Generic.Deleted */
|
||||
.ge { color: #586e75; font-style: italic } /* Generic.Emph */
|
||||
.gr { color: #dc322f } /* Generic.Error */
|
||||
.gh { color: #cb4b16 } /* Generic.Heading */
|
||||
.gi { color: #859900 } /* Generic.Inserted */
|
||||
.go { color: #586e75 } /* Generic.Output */
|
||||
.gp { color: #586e75 } /* Generic.Prompt */
|
||||
.gs { color: #586e75; font-weight: bold } /* Generic.Strong */
|
||||
.gu { color: #cb4b16 } /* Generic.Subheading */
|
||||
.gt { color: #586e75 } /* Generic.Traceback */
|
||||
.kc { color: #cb4b16 } /* Keyword.Constant */
|
||||
.kd { color: #268bd2 } /* Keyword.Declaration */
|
||||
.kn { color: #859900 } /* Keyword.Namespace */
|
||||
.kp { color: #859900 } /* Keyword.Pseudo */
|
||||
.kr { color: #268bd2 } /* Keyword.Reserved */
|
||||
.kt { color: #dc322f } /* Keyword.Type */
|
||||
.ld { color: #586e75 } /* Literal.Date */
|
||||
.m { color: #2aa198 } /* Literal.Number */
|
||||
.s { color: #2aa198 } /* Literal.String */
|
||||
.na { color: #586e75 } /* Name.Attribute */
|
||||
.nb { color: #B58900 } /* Name.Builtin */
|
||||
.nc { color: #268bd2 } /* Name.Class */
|
||||
.no { color: #cb4b16 } /* Name.Constant */
|
||||
.nd { color: #268bd2 } /* Name.Decorator */
|
||||
.ni { color: #cb4b16 } /* Name.Entity */
|
||||
.ne { color: #cb4b16 } /* Name.Exception */
|
||||
.nf { color: #268bd2 } /* Name.Function */
|
||||
.nl { color: #586e75 } /* Name.Label */
|
||||
.nn { color: #586e75 } /* Name.Namespace */
|
||||
.nx { color: #586e75 } /* Name.Other */
|
||||
.py { color: #586e75 } /* Name.Property */
|
||||
.nt { color: #268bd2 } /* Name.Tag */
|
||||
.nv { color: #268bd2 } /* Name.Variable */
|
||||
.ow { color: #859900 } /* Operator.Word */
|
||||
.w { color: #586e75 } /* Text.Whitespace */
|
||||
.mf { color: #2aa198 } /* Literal.Number.Float */
|
||||
.mh { color: #2aa198 } /* Literal.Number.Hex */
|
||||
.mi { color: #2aa198 } /* Literal.Number.Integer */
|
||||
.mo { color: #2aa198 } /* Literal.Number.Oct */
|
||||
.sb { color: #93a1a1 } /* Literal.String.Backtick */
|
||||
.sc { color: #2aa198 } /* Literal.String.Char */
|
||||
.sd { color: #586e75 } /* Literal.String.Doc */
|
||||
.s2 { color: #2aa198 } /* Literal.String.Double */
|
||||
.se { color: #cb4b16 } /* Literal.String.Escape */
|
||||
.sh { color: #586e75 } /* Literal.String.Heredoc */
|
||||
.si { color: #2aa198 } /* Literal.String.Interpol */
|
||||
.sx { color: #2aa198 } /* Literal.String.Other */
|
||||
.sr { color: #dc322f } /* Literal.String.Regex */
|
||||
.s1 { color: #2aa198 } /* Literal.String.Single */
|
||||
.ss { color: #2aa198 } /* Literal.String.Symbol */
|
||||
.bp { color: #268bd2 } /* Name.Builtin.Pseudo */
|
||||
.vc { color: #268bd2 } /* Name.Variable.Class */
|
||||
.vg { color: #268bd2 } /* Name.Variable.Global */
|
||||
.vi { color: #268bd2 } /* Name.Variable.Instance */
|
||||
.il { color: #2aa198 } /* Literal.Number.Integer.Long */
|
||||
}
|
||||
|
||||
div.epigraph {
|
||||
@@ -319,17 +363,6 @@ div.clear {
|
||||
clear: both;
|
||||
}
|
||||
|
||||
.sans {
|
||||
font-family: $sans;
|
||||
}
|
||||
|
||||
.exercise {
|
||||
background: rgba(38, 139, 210, 0.1);
|
||||
border-radius: 10px;
|
||||
margin: 0px -20px;
|
||||
padding: 5px 20px;
|
||||
}
|
||||
|
||||
.admonition {
|
||||
.admonition-title {
|
||||
font-weight: bold;
|
||||
@@ -356,12 +389,14 @@ div.clear {
|
||||
.admonition.tryit {
|
||||
background: rgba(185, 246, 202, 1);
|
||||
|
||||
.codehilite {
|
||||
background-color: transparent;
|
||||
color: inherit;
|
||||
margin: 0px 0px;
|
||||
.highlight {
|
||||
pre {
|
||||
background-color: transparent;
|
||||
color: inherit;
|
||||
margin: 0px 0px;
|
||||
|
||||
.c { color: grey; }
|
||||
.c { color: grey; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
<link href="https://fonts.googleapis.com/css?family=Montserrat" rel="stylesheet">
|
||||
<link rel="stylesheet" href="css/style.css" type="text/css">
|
||||
<link rel="apple-touch-icon" sizes="57x57" href="images/favicon/apple-icon-57x57.png">
|
||||
<link rel="apple-touch-icon" sizes="60x60" href="images/favicon/apple-icon-60x60.png">
|
||||
<link rel="apple-touch-icon" sizes="72x72" href="images/favicon/apple-icon-72x72.png">
|
||||
<link rel="apple-touch-icon" sizes="76x76" href="images/favicon/apple-icon-76x76.png">
|
||||
<link rel="apple-touch-icon" sizes="114x114" href="images/favicon/apple-icon-114x114.png">
|
||||
<link rel="apple-touch-icon" sizes="120x120" href="images/favicon/apple-icon-120x120.png">
|
||||
<link rel="apple-touch-icon" sizes="144x144" href="images/favicon/apple-icon-144x144.png">
|
||||
<link rel="apple-touch-icon" sizes="152x152" href="images/favicon/apple-icon-152x152.png">
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="images/favicon/apple-icon-180x180.png">
|
||||
<link rel="icon" type="image/png" sizes="192x192" href="images/favicon/android-icon-192x192.png">
|
||||
<link rel="icon" type="image/png" sizes="32x32" href="images/favicon/favicon-32x32.png">
|
||||
<link rel="icon" type="image/png" sizes="96x96" href="images/favicon/favicon-96x96.png">
|
||||
<link rel="icon" type="image/png" sizes="16x16" href="images/favicon/favicon-16x16.png">
|
||||
@@ -2,9 +2,9 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta charset="utf-8">
|
||||
<title>Dart VM</title>
|
||||
<link rel="stylesheet" href="/css/style.css" type="text/css">
|
||||
{% include 'includes/favicon.html' %}
|
||||
<link rel="stylesheet" href="{{root}}/css/style.css" type="text/css">
|
||||
{% include 'includes/auto-refresh.html' %}
|
||||
</head>
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
# Files and directories created by pub
|
||||
.dart_tool/
|
||||
.packages
|
||||
# Remove the following pattern if you wish to check in your lock file
|
||||
pubspec.lock
|
||||
|
||||
# Conventional directory for build outputs
|
||||
build/
|
||||
|
||||
# Directory created by dartdoc
|
||||
doc/api/
|
||||
@@ -1,54 +0,0 @@
|
||||
Tool for extracting symbolic information from C++ Dart Runtime sources using
|
||||
[cquery](https://github.com/cquery-project/cquery).
|
||||
|
||||
It should be invoked from the root of Dart SDK checkout and will generate
|
||||
`xref.json` file containing extracted symbol information.
|
||||
|
||||
```
|
||||
$ pushd runtime/tools/wiki/xref_extractor && pub get && popd
|
||||
$ dart runtime/tools/wiki/xref_extractor/bin/main.dart cquery/build/release/bin/cquery
|
||||
```
|
||||
|
||||
# Prerequisites
|
||||
|
||||
1. Build [cquery](https://github.com/cquery-project/cquery) as described [here](https://github.com/cquery-project/cquery/wiki/Building-cquery).
|
||||
2. Make sure that you have ninja files generated for ReleaseX64 configuration by
|
||||
running `tools/gn.py -a x64 -m release --no-goma` (`--no-goma` is important -
|
||||
otherwise `cquery` can't figure out which toolchain is used).
|
||||
|
||||
# `xref.json` format
|
||||
|
||||
```typescript
|
||||
interface Xrefs {
|
||||
/// Commit hash for which this xref.json is generated.
|
||||
commit: string;
|
||||
|
||||
/// List of files names.
|
||||
files: string[];
|
||||
|
||||
/// Class information by name.
|
||||
classes: ClassMap;
|
||||
|
||||
/// Global function information.
|
||||
functions: LocationMap;
|
||||
}
|
||||
|
||||
/// Locations are serialized as strings of form "fileIndex:lineNo", where
|
||||
/// fileIndex points into files array.
|
||||
type SymbolLocation = string;
|
||||
|
||||
/// Information about classes is stored in an array where the first element
|
||||
/// describes location of the class itself and second optional element gives
|
||||
/// locations of class members.
|
||||
type ClassInfo = [SymbolLocation, LocationMap?];
|
||||
|
||||
/// Map of classes by their names.
|
||||
interface ClassMap {
|
||||
[name: string]: ClassInfo;
|
||||
}
|
||||
|
||||
/// Map of symbols to their locations.
|
||||
interface LocationMap {
|
||||
[symbol: string]: SymbolLocation;
|
||||
}
|
||||
```
|
||||
@@ -1 +0,0 @@
|
||||
include: package:lints/core.yaml
|
||||
@@ -1,107 +0,0 @@
|
||||
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
// Script used to extract symbols with locations from runtime/vm files using
|
||||
// cquery. See README.md for more information.
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:path/path.dart' as p;
|
||||
|
||||
import 'package:xref_extractor/cquery_driver.dart';
|
||||
import 'package:xref_extractor/xref_extractor.dart';
|
||||
|
||||
// Note: not using Directory.createTemp to reduce indexing costs.
|
||||
const cqueryCachePath = '/tmp/cquery-cache-for-dart-sdk';
|
||||
|
||||
void main(List<String> args) async {
|
||||
if (args.length != 1 || !File(args[0]).existsSync()) {
|
||||
print('''
|
||||
Usage: dart runtime/tools/wiki/xref_extractor/bin/main.dart <path-to-cquery>
|
||||
''');
|
||||
exit(1);
|
||||
}
|
||||
|
||||
final cqueryBinary = args[0];
|
||||
|
||||
// Sanity check that we are running from SDK checkout root.
|
||||
final sdkCheckoutRoot = Directory.current.absolute;
|
||||
final runtimeVmDirectory =
|
||||
Directory(p.join(sdkCheckoutRoot.path, 'runtime/vm'));
|
||||
final gitDirectory = Directory(p.join(sdkCheckoutRoot.path, '.git'));
|
||||
|
||||
if (!gitDirectory.existsSync() || !runtimeVmDirectory.existsSync()) {
|
||||
print('This script expects to be run from SDK checkout root');
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Generate compile_commands.json from which cquery will extract compilation
|
||||
// flags for individual C++ files.
|
||||
await generateCompileCommands();
|
||||
|
||||
// Start cquery process and request indexing of runtimeVmDirectory.
|
||||
final cquery = await CqueryDriver.start(cqueryBinary);
|
||||
|
||||
print('Indexing ${runtimeVmDirectory.path} with cquery');
|
||||
cquery.progress.listen((files) =>
|
||||
stdout.write('\rcquery is running ($files files left to index)'));
|
||||
|
||||
await cquery.request('initialize', params: {
|
||||
'processId': 123,
|
||||
'rootUri': sdkCheckoutRoot.uri.toString(),
|
||||
'capabilities': {
|
||||
'textDocument': {'codeLens': null}
|
||||
},
|
||||
'trace': 'on',
|
||||
'initializationOptions': {
|
||||
'cacheDirectory': cqueryCachePath,
|
||||
'progressReportFrequencyMs': 1000,
|
||||
},
|
||||
'workspaceFolders': [
|
||||
{
|
||||
'uri': runtimeVmDirectory.uri.toString(),
|
||||
'name': 'vm',
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
// Tell cquery to wait for the indexing to complete and then exit.
|
||||
cquery.notify(r'$cquery/wait');
|
||||
cquery.notify(r'exit');
|
||||
|
||||
// Wait for cquery to exit.
|
||||
final exitCode = await cquery.exitCode;
|
||||
print('\r\x1b[K... completed (cquery exited with exit code ${exitCode})');
|
||||
|
||||
// Process cquery cache folder to extract symbolic information.
|
||||
await generateXRef(cqueryCachePath, sdkCheckoutRoot.path,
|
||||
(path) => path.startsWith('runtime/'));
|
||||
}
|
||||
|
||||
/// Generate compile_commands.json for cquery so that it could index VM sources.
|
||||
///
|
||||
/// We ask ninja to produce compilation database for X64 release build and then
|
||||
/// post process it to limit it to libdart_vm_precompiler_host_targeting_host
|
||||
/// target, because otherwise we get duplicated compilation commands for the
|
||||
/// same input C++ files and this greatly confuses cquery.
|
||||
Future<void> generateCompileCommands() async {
|
||||
print('Extracting compilation commands from build files for ReleaseX64');
|
||||
final result = await Process.run('buildtools/ninja/ninja', [
|
||||
'-C',
|
||||
'${Platform.isMacOS ? 'xcodebuild' : 'out'}/ReleaseX64',
|
||||
'-t',
|
||||
'compdb',
|
||||
'cxx'
|
||||
]);
|
||||
final List<dynamic> commands = jsonDecode(result.stdout);
|
||||
final re = RegExp(r'/libdart(_vm)?_precompiler_host_targeting_host\.');
|
||||
final filteredCommands = commands
|
||||
.cast<Map<String, dynamic>>()
|
||||
.where((item) => item['command'].contains(re))
|
||||
.toList(growable: false);
|
||||
File('compile_commands.json').writeAsStringSync(jsonEncode(filteredCommands));
|
||||
print('''
|
||||
... generated compile_commands.json with ${filteredCommands.length} entries''');
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
// Library for spawning cquery process and communicating with it.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:math' as math;
|
||||
|
||||
/// Helper class encapsulating communication with the cquery process.
|
||||
///
|
||||
/// cquery communicates using jsonrpc via stdin/stdout streams.
|
||||
class CqueryDriver {
|
||||
final Process _cquery;
|
||||
|
||||
final _pendingRequests = <int, Completer<dynamic>>{};
|
||||
var _requestId = 1;
|
||||
|
||||
final _progressStreamController = StreamController<int>.broadcast();
|
||||
|
||||
CqueryDriver._(this._cquery) {
|
||||
_cquery.stdout
|
||||
.transform(utf8.decoder)
|
||||
.transform(_JsonRpcParser.transformer)
|
||||
.listen(_handleMessage);
|
||||
_cquery.stderr
|
||||
.transform(utf8.decoder)
|
||||
.listen((data) => print('stderr: ${data}'));
|
||||
}
|
||||
|
||||
static Future<CqueryDriver> start(String cqueryBinary) async {
|
||||
final cquery = await Process.start(cqueryBinary, ['--language-server']);
|
||||
return CqueryDriver._(cquery);
|
||||
}
|
||||
|
||||
/// Stream of progress notifications from cquery. Indicate how many files
|
||||
/// are currently pending indexing.
|
||||
Stream<int> get progress => _progressStreamController.stream;
|
||||
|
||||
Future<int> get exitCode => _cquery.exitCode;
|
||||
|
||||
/// Send the given message to the cquery process.
|
||||
void _sendMessage(Map<String, dynamic> data) {
|
||||
data['jsonrpc'] = '2.0';
|
||||
|
||||
final rq = jsonEncode(data);
|
||||
_cquery.stdin.write('Content-Length: ${rq.length}\r\n\r\n${rq}');
|
||||
}
|
||||
|
||||
/// Send a notification message to the cquery process.
|
||||
void notify(String method, {Map<String, dynamic> params}) {
|
||||
final rq = <String, dynamic>{'method': method};
|
||||
if (params != null) rq['params'] = params;
|
||||
_sendMessage(rq);
|
||||
}
|
||||
|
||||
/// Send a request message to the cquery process.
|
||||
Future<dynamic> request(String method, {Map<String, dynamic> params}) {
|
||||
final rq = <String, dynamic>{'method': method, 'id': _requestId++};
|
||||
_pendingRequests[rq['id']] = Completer();
|
||||
if (params != null) rq['params'] = params;
|
||||
_sendMessage(rq);
|
||||
return _pendingRequests[rq['id']].future;
|
||||
}
|
||||
|
||||
/// Handle message received from cquery process.
|
||||
void _handleMessage(Map<String, dynamic> data) {
|
||||
final method = data['method'];
|
||||
|
||||
// If it is a progress notification issue progress event.
|
||||
if (method == r'$cquery/progress') {
|
||||
_progressStreamController.add(data['params']['indexRequestCount']);
|
||||
}
|
||||
|
||||
// Otherwise check if it is a response to one of our requests and complete
|
||||
// corresponding future if it is.
|
||||
if (data.containsKey('id') && data.containsKey('result')) {
|
||||
final id = data['id'];
|
||||
final result = data['result'];
|
||||
_pendingRequests[id].complete(result);
|
||||
_pendingRequests[id] = null;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Simple parser for jsonrpc protocol over arbitrary chunked stream.
|
||||
class _JsonRpcParser {
|
||||
/// Accumulator for the message content.
|
||||
final StringBuffer content = StringBuffer();
|
||||
|
||||
/// Number of bytes left to read in the current message.
|
||||
int pendingContentLength = 0;
|
||||
|
||||
/// Auxiliary variable to store various state between invocations of
|
||||
/// [state] callback.
|
||||
int index = 0;
|
||||
|
||||
/// Position inside incoming chunk of data.
|
||||
int pos = 0;
|
||||
|
||||
/// Current parser state.
|
||||
Function state = _JsonRpcParser.readHeader;
|
||||
|
||||
/// Callback to invoke when we finish parsing complete message.
|
||||
void Function(Map<String, dynamic>) onMessage;
|
||||
|
||||
_JsonRpcParser({this.onMessage});
|
||||
|
||||
/// StreamTransformer wrapping _JsonRpcParser.
|
||||
static StreamTransformer<String, Map<String, dynamic>> get transformer =>
|
||||
StreamTransformer.fromBind((Stream<String> s) {
|
||||
final output = StreamController<Map<String, dynamic>>();
|
||||
final p = _JsonRpcParser(onMessage: output.add);
|
||||
s.listen(p.addChunk);
|
||||
return output.stream;
|
||||
});
|
||||
|
||||
/// Parse the chunk of data.
|
||||
void addChunk(String data) {
|
||||
pos = 0;
|
||||
while (pos < data.length) {
|
||||
state = state(this, data);
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsing state: waiting for 'Content-Length' header.
|
||||
static Function readHeader(_JsonRpcParser p, String data) {
|
||||
final codeUnit = data.codeUnitAt(p.pos++);
|
||||
|
||||
if (HEADER.codeUnitAt(p.index) != codeUnit) {
|
||||
throw 'Unexpected codeUnit: ${String.fromCharCode(codeUnit)} expected ${HEADER[p.index]}';
|
||||
}
|
||||
|
||||
p.index++;
|
||||
if (p.index == HEADER.length) {
|
||||
p.index = 0;
|
||||
return _JsonRpcParser.readLength;
|
||||
}
|
||||
return _JsonRpcParser.readHeader;
|
||||
}
|
||||
|
||||
/// Parsing state: parsing content length value.
|
||||
static Function readLength(_JsonRpcParser p, String data) {
|
||||
final codeUnit = data.codeUnitAt(p.pos++);
|
||||
|
||||
if (codeUnit == CR) {
|
||||
p.pendingContentLength = p.index;
|
||||
p.index = 0;
|
||||
return _JsonRpcParser.readHeaderEnd;
|
||||
}
|
||||
if (codeUnit < CH0 || codeUnit > CH9) {
|
||||
throw 'Unexpected codeUnit: ${String.fromCharCode(codeUnit)} expected 0 to 9';
|
||||
}
|
||||
p.index = p.index * 10 + (codeUnit - CH0);
|
||||
return _JsonRpcParser.readLength;
|
||||
}
|
||||
|
||||
/// Parsing state: content length was read, skipping line breaks before
|
||||
/// content start.
|
||||
static Function readHeaderEnd(_JsonRpcParser p, String data) {
|
||||
final codeUnit = data.codeUnitAt(p.pos++);
|
||||
|
||||
if (HEADER_END.codeUnitAt(p.index) != codeUnit) {
|
||||
throw 'Unexpected codeUnit: ${String.fromCharCode(codeUnit)} expected ${HEADER_END[p.index]}';
|
||||
}
|
||||
|
||||
p.index++;
|
||||
if (p.index == HEADER_END.length) {
|
||||
return _JsonRpcParser.readContent;
|
||||
}
|
||||
return _JsonRpcParser.readHeaderEnd;
|
||||
}
|
||||
|
||||
/// Parsing state: reading message content.
|
||||
static Function readContent(_JsonRpcParser p, String data) {
|
||||
final availableBytes = data.length - p.pos;
|
||||
final bytesToRead = math.min(availableBytes, p.pendingContentLength);
|
||||
p.content.write(data.substring(p.pos, p.pos + bytesToRead));
|
||||
p.pendingContentLength -= bytesToRead;
|
||||
p.pos += bytesToRead;
|
||||
if (p.pendingContentLength == 0) {
|
||||
p.onMessage(jsonDecode(p.content.toString()));
|
||||
p.content.clear();
|
||||
p.index = 0;
|
||||
return _JsonRpcParser.readHeader;
|
||||
} else {
|
||||
return _JsonRpcParser.readContent;
|
||||
}
|
||||
}
|
||||
|
||||
static const HEADER = 'Content-Length: ';
|
||||
static const HEADER_END = '\n\r\n';
|
||||
static const CH0 = 48;
|
||||
static const CH9 = 57;
|
||||
static const CR = 13;
|
||||
static const LF = 10; // ignore: unused_field
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
// Library for converting cquery cache into condensed symbol location
|
||||
// information expected by our xref markdown extension.
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:path/path.dart' as p;
|
||||
|
||||
typedef FileFilterCallback = bool Function(String path);
|
||||
|
||||
/// Load cquery cache from the given directory and accumulate all symbols
|
||||
/// (global functions and class methods) for the Dart SDK related sources
|
||||
/// matching the given filter.
|
||||
Future<void> generateXRef(String cqueryCachePath, String sdkRootPath,
|
||||
FileFilterCallback fileFilter) async {
|
||||
final cacheRoot =
|
||||
Directory(p.join(cqueryCachePath, sdkRootPath.replaceAll('/', '@')));
|
||||
final files = cacheRoot
|
||||
.listSync()
|
||||
.whereType<File>()
|
||||
.where((file) => file.path.endsWith('.json'))
|
||||
.toList();
|
||||
print('Processing ${files.length} indexes available in ${cacheRoot.path}');
|
||||
|
||||
final cache = CqueryCache(fileFilter: fileFilter);
|
||||
files.forEach(cache.loadFile);
|
||||
|
||||
final classesByName = <String, Class>{
|
||||
for (var entry in cache.classes.entries)
|
||||
if (entry.value.name != null && entry.value.name != '')
|
||||
entry.value.name: entry.value
|
||||
};
|
||||
|
||||
final database = {
|
||||
'commit': await currentCommitHash(),
|
||||
'files': cache.filesByIndex,
|
||||
'classes': classesByName,
|
||||
'functions': cache.globals.uniqueMembers
|
||||
};
|
||||
|
||||
File('xref.json').writeAsStringSync(jsonEncode(database));
|
||||
print('... done (written xref.json)');
|
||||
}
|
||||
|
||||
/// Helper class representing symbol information contained in the cquery cache.
|
||||
class CqueryCache {
|
||||
final files = <String, int>{};
|
||||
final filesByIndex = [];
|
||||
|
||||
final classes = <num, Class>{};
|
||||
final globals = Class('\$Globals');
|
||||
|
||||
FileFilterCallback fileFilter;
|
||||
|
||||
CqueryCache({this.fileFilter});
|
||||
|
||||
int addFile(String name) => files.putIfAbsent(name, () {
|
||||
filesByIndex.add(name);
|
||||
return filesByIndex.length - 1;
|
||||
});
|
||||
|
||||
Location makeLocation(String file, int lineNo) =>
|
||||
Location(addFile(file), lineNo);
|
||||
|
||||
// cquery used to serialize USRs as integers but they are now serialized as
|
||||
// doubles (with .0 at the end) for some reason. This might even lead to
|
||||
// incorrect deserialization with a loss of a precise USR value - but
|
||||
// should not lead to any issues as long as two different classes don't
|
||||
// have conflicting USRs.
|
||||
Class findClassByUsr(num usr) => classes.putIfAbsent(usr, () => Class());
|
||||
|
||||
void defineClass(num usr, String name, Location loc) {
|
||||
final cls = findClassByUsr(usr);
|
||||
if (cls.name != null && cls.name != '' && cls.name != name) {
|
||||
throw 'Mismatched names';
|
||||
}
|
||||
if (name != '') cls.name = name;
|
||||
if (cls.loc == null) {
|
||||
cls.loc = loc;
|
||||
} else {
|
||||
cls.loc = Location.invalid;
|
||||
}
|
||||
}
|
||||
|
||||
void loadFile(File indexFile) {
|
||||
final result = jsonDecode(indexFile.readAsStringSync().split('\n')[1]);
|
||||
|
||||
// Check if we are interested in the original source file.
|
||||
final sourceFile =
|
||||
p.basenameWithoutExtension(indexFile.path).replaceAll('@', '/');
|
||||
if (!fileFilter(sourceFile)) return;
|
||||
|
||||
// Extract classes defined in the file.
|
||||
for (var type in result['types']) {
|
||||
if (type['kind'] != SymbolKind.Class) continue;
|
||||
final extent = type['extent'];
|
||||
if (extent == null) continue;
|
||||
|
||||
final detailedName = type['detailed_name'];
|
||||
final lineStart = int.parse(extent.substring(0, extent.indexOf(':')));
|
||||
defineClass(
|
||||
type['usr'], detailedName, makeLocation(sourceFile, lineStart));
|
||||
}
|
||||
|
||||
// Extract class methods defined in the file.
|
||||
for (var func in result['funcs']) {
|
||||
final kind = func['kind'];
|
||||
if (kind != SymbolKind.Method && kind != SymbolKind.StaticMethod) {
|
||||
continue;
|
||||
}
|
||||
final extent = func['extent'];
|
||||
if (extent == null) continue;
|
||||
final short = shortName(func);
|
||||
final lineStart = int.parse(extent.substring(0, extent.indexOf(':')));
|
||||
if (func['declaring_type'] == null) {
|
||||
continue;
|
||||
}
|
||||
findClassByUsr(result['types'][func['declaring_type']]['usr'])
|
||||
.defineMember(short, makeLocation(sourceFile, lineStart));
|
||||
}
|
||||
|
||||
// Extract global functions defined in the file.
|
||||
for (var func in result['funcs']) {
|
||||
final kind = func['kind'];
|
||||
if (kind != SymbolKind.Function) continue;
|
||||
final extent = func['extent'];
|
||||
if (extent == null) continue;
|
||||
final short = shortName(func);
|
||||
final lineStart = int.parse(extent.substring(0, extent.indexOf(':')));
|
||||
globals.defineMember(short, makeLocation(sourceFile, lineStart));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class Class {
|
||||
String name;
|
||||
Location loc;
|
||||
|
||||
// Member to definition location map. If the same symbol has multiple
|
||||
// definitions then we mark it with [Location.invalid].
|
||||
Map<String, Location> members;
|
||||
|
||||
Class([this.name]);
|
||||
|
||||
void defineMember(String name, Location loc) {
|
||||
members ??= <String, Location>{};
|
||||
members[name] = members.containsKey(name) ? Location.invalid : loc;
|
||||
}
|
||||
|
||||
dynamic toJson() {
|
||||
final result = [loc?.toJson() ?? 0];
|
||||
if (members != null) {
|
||||
final res = uniqueMembers;
|
||||
if (res.isNotEmpty) {
|
||||
result.add(res);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Map<String, Location> get uniqueMembers => <String, Location>{
|
||||
for (var entry in members.entries)
|
||||
if (entry.value != Location.invalid) entry.key: entry.value
|
||||
};
|
||||
}
|
||||
|
||||
class Location {
|
||||
final int file;
|
||||
final int lineNo;
|
||||
|
||||
const Location(this.file, this.lineNo);
|
||||
|
||||
String toJson() => identical(this, invalid) ? null : '$file:$lineNo';
|
||||
|
||||
@override
|
||||
String toString() => '$file:$lineNo';
|
||||
|
||||
static const invalid = Location(-1, -1);
|
||||
}
|
||||
|
||||
String shortName(entity) {
|
||||
final offset = entity['short_name_offset'];
|
||||
final length = entity['short_name_size'] ?? 0;
|
||||
final detailedName = entity['detailed_name'];
|
||||
if (length == 0) return detailedName;
|
||||
return detailedName.substring(offset, offset + length);
|
||||
}
|
||||
|
||||
Future<String> currentCommitHash() async {
|
||||
final results = await Process.run('git', ['rev-parse', 'HEAD']);
|
||||
return results.stdout;
|
||||
}
|
||||
|
||||
/// Kind of the symbol. Taken from LSP specifications and cquery source code.
|
||||
abstract class SymbolKind {
|
||||
static const Unknown = 0;
|
||||
|
||||
static const File = 1;
|
||||
static const Module = 2;
|
||||
static const Namespace = 3;
|
||||
static const Package = 4;
|
||||
static const Class = 5;
|
||||
static const Method = 6;
|
||||
static const Property = 7;
|
||||
static const Field = 8;
|
||||
static const Constructor = 9;
|
||||
static const Enum = 10;
|
||||
static const Interface = 11;
|
||||
static const Function = 12;
|
||||
static const Variable = 13;
|
||||
static const Constant = 14;
|
||||
static const String = 15;
|
||||
static const Number = 16;
|
||||
static const Boolean = 17;
|
||||
static const Array = 18;
|
||||
static const Object = 19;
|
||||
static const Key = 20;
|
||||
static const Null = 21;
|
||||
static const EnumMember = 22;
|
||||
static const Struct = 23;
|
||||
static const Event = 24;
|
||||
static const Operator = 25;
|
||||
static const TypeParameter = 26;
|
||||
|
||||
// cquery extensions.
|
||||
static const TypeAlias = 252;
|
||||
static const Parameter = 253;
|
||||
static const StaticMethod = 254;
|
||||
static const Macro = 255;
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
name: xref_extractor
|
||||
description: A sample command-line application.
|
||||
# This package is not intended for consumption on pub.dev. DO NOT publish.
|
||||
publish_to: none
|
||||
|
||||
environment:
|
||||
sdk: '>=2.7.0 <3.0.0'
|
||||
|
||||
dependencies:
|
||||
path: any
|
||||
|
||||
dev_dependencies:
|
||||
lints: any
|
||||
Reference in New Issue
Block a user