[analysis_server] Migrate from using Markdown/TypeScript spec for LSP types to JSON model

Change-Id: I58dbbbee48febc45304b27a95fedfef289479265
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/247340
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
This commit is contained in:
Danny Tuppeny
2022-06-06 14:27:48 +00:00
committed by Commit Bot
parent 2deae5a599
commit 086727ee6f
13 changed files with 17193 additions and 1907 deletions
File diff suppressed because it is too large Load Diff
@@ -11,8 +11,13 @@ import 'package:analysis_server/src/services/refactoring/refactoring.dart';
import 'package:analysis_server/src/services/refactoring/rename_unit_member.dart';
import 'package:analyzer/dart/element/element.dart';
// TODO(dantup): Generate typedefs in protocol_generated for all named types
// that map onto unions like this after the switch to JSON spec.
typedef PrepareRenameResult
= Either3<Range, PlaceholderAndRange, PrepareRenameResult2>;
class PrepareRenameHandler
extends MessageHandler<TextDocumentPositionParams, PlaceholderAndRange?> {
extends MessageHandler<TextDocumentPositionParams, PrepareRenameResult?> {
PrepareRenameHandler(super.server);
@override
Method get handlesMessage => Method.textDocument_prepareRename;
@@ -22,7 +27,7 @@ class PrepareRenameHandler
TextDocumentPositionParams.jsonHandler;
@override
Future<ErrorOr<PlaceholderAndRange?>> handle(
Future<ErrorOr<PrepareRenameResult?>> handle(
TextDocumentPositionParams params,
MessageInfo message,
CancellationToken token) async {
@@ -61,7 +66,7 @@ class PrepareRenameHandler
ServerErrorCodes.RenameNotValid, initStatus.problem!.message, null);
}
return success(PlaceholderAndRange(
return success(PrepareRenameResult.t2(PlaceholderAndRange(
range: toRange(
unit.result.lineInfo,
// If the offset is set to -1 it means there is no location for the
@@ -72,7 +77,7 @@ class PrepareRenameHandler
refactorDetails.length,
),
placeholder: refactoring.oldName,
));
)));
});
}
}
@@ -38,6 +38,7 @@ class ColorComputerTest extends AbstractContextTest {
'Color(0xFF0000FF)': 0xFF0000FF,
'Color.fromARGB(255, 0, 0, 255)': 0xFF0000FF,
'Color.fromRGBO(0, 0, 255, 1)': 0xFF0000FF,
'Color.fromRGBO(0, 0, 255, 1.0)': 0xFF0000FF,
// Flutter Painting
'ColorSwatch(0xFF89ABCD, {})': 0xFF89ABCD,
// Flutter Material
@@ -66,7 +66,7 @@ class LiteralTypeMatcher extends Matcher {
bool matches(item, Map matchState) {
return item is LiteralType &&
_typeMatcher.matches(item.type, matchState) &&
item.literal == _value;
item.valueAsLiteral == _value;
}
}
@@ -8,20 +8,30 @@ import '../../../tool/lsp_spec/typescript_parser.dart';
import 'matchers.dart';
void main() {
group('typescript parser', () {
test('parses an interface', () {
final input = '''
/**
* Some options.
*/
export interface SomeOptions {
/**
* Options used by something.
*/
options?: OptionKind[];
}
''';
final output = parseString(input);
// TODO(dantup): Rename this file in a seperate CL so it doesn't lose its
// history because the number of the large number of changes.
group('meta model reader', () {
test('reads an interface', () {
final input = {
"structures": [
{
"name": "SomeOptions",
"properties": [
{
"name": "options",
"type": {
"kind": "array",
"element": {"kind": "reference", "name": "string"}
},
"optional": true,
"documentation": "Options used by something.",
}
],
"documentation": "Some options."
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
@@ -35,18 +45,37 @@ export interface SomeOptions {
expect(field.commentText, equals('''Options used by something.'''));
expect(field.allowsNull, isFalse);
expect(field.allowsUndefined, isTrue);
expect(field.type, isArrayOf(isSimpleType('OptionKind')));
expect(field.type, isArrayOf(isSimpleType('string')));
});
test('parses an interface with a field with an inline/unnamed type', () {
final input = '''
export interface Capabilities {
textDoc?: {
deprecated?: bool;
};
}
''';
final output = parseString(input);
test('reads an interface with a field with an inline/unnamed type', () {
final input = {
"structures": [
{
"name": "Capabilities",
"properties": [
{
"name": "textDoc",
"type": {
"kind": "literal",
"value": {
"properties": [
{
"name": "deprecated",
"type": {"kind": "base", "name": "bool"},
"optional": true,
}
]
}
},
"optional": true,
}
],
"documentation": "Some options."
},
],
};
final output = readModel(input);
// Length is two because we'll fabricate the type of textDoc.
expect(output, hasLength(2));
@@ -74,20 +103,27 @@ export interface Capabilities {
expect(field.type, isSimpleType('CapabilitiesTextDoc'));
});
test('parses an interface with multiple fields', () {
final input = '''
export interface SomeOptions {
/**
* Options0 used by something.
*/
options0: any;
/**
* Options1 used by something.
*/
options1: any;
}
''';
final output = parseString(input);
test('reads an interface with multiple fields', () {
final input = {
"structures": [
{
"name": "SomeOptions",
"properties": [
{
"name": "options0",
"type": {"kind": "reference", "name": "LSPAny"},
"documentation": "Options0 used by something.",
},
{
"name": "options1",
"type": {"kind": "reference", "name": "LSPAny"},
"documentation": "Options1 used by something.",
}
],
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
@@ -100,59 +136,28 @@ export interface SomeOptions {
}
});
test('parses an interface with type args', () {
final input = '''
interface MyInterface<D> {
data?: D;
}
''';
final output = parseString(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
expect(interface.members, hasLength(1));
final field = interface.members.first as Field;
expect(field, const TypeMatcher<Field>());
expect(field.name, equals('data'));
expect(field.allowsUndefined, isTrue);
expect(field.allowsNull, isFalse);
expect(field.type, isSimpleType('D'));
});
test('parses an interface with Arrays in Array<T> format', () {
final input = '''
export interface MyMessage {
/**
* The method's params.
*/
params?: Array<any> | string;
}
''';
final output = parseString(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
expect(interface.members, hasLength(1));
final field = interface.members.first as Field;
expect(field, const TypeMatcher<Field>());
expect(field.name, equals('params'));
expect(field.commentText, equals('''The method's params.'''));
expect(field.allowsUndefined, isTrue);
expect(field.allowsNull, isFalse);
expect(field.type, const TypeMatcher<UnionType>());
final union = field.type as UnionType;
expect(union.types, hasLength(2));
expect(union.types[0], isArrayOf(isSimpleType('any')));
expect(union.types[1], isSimpleType('string'));
});
test('parses an interface with a map into a MapType', () {
final input = '''
export interface WorkspaceEdit {
changes: { [uri: string]: TextEdit[]; };
}
''';
final output = parseString(input);
test('reads an interface with a map into a MapType', () {
final input = {
"structures": [
{
"name": "WorkspaceEdit",
"properties": [
{
"name": "changes",
"type": {
"kind": "map",
"key": {"kind": "base", "name": "string"},
"value": {
"kind": "array",
"element": {"kind": "reference", "name": "TextEdit"}
},
},
}
],
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
@@ -165,15 +170,46 @@ export interface WorkspaceEdit {
});
test('flags nullable undefined values', () {
final input = '''
export interface A {
canBeBoth?: string | null;
canBeNeither: string;
canBeNull: string | null;
canBeUndefined?: string;
}
''';
final output = parseString(input);
final input = {
"structures": [
{
"name": "A",
"properties": [
{
"name": "canBeBoth",
"type": {
"kind": "or",
"items": [
{"kind": "base", "name": "string"},
{"kind": "base", "name": "null"}
]
},
"optional": true,
},
{
"name": "canBeNeither",
"type": {"kind": "base", "name": "string"},
},
{
"name": "canBeNull",
"type": {
"kind": "or",
"items": [
{"kind": "base", "name": "string"},
{"kind": "base", "name": "null"}
]
},
},
{
"name": "canBeUndefined",
"type": {"kind": "base", "name": "string"},
"optional": true,
},
],
},
],
};
final output = readModel(input);
final interface = output[0] as Interface;
expect(interface.members, hasLength(4));
for (var m in interface.members) {
@@ -194,28 +230,28 @@ export interface A {
});
test('formats comments correctly', () {
final input = '''
/**
* Describes the what this class in lots of words that wrap onto
* multiple lines that will need re-wrapping to format nicely when
* converted into Dart.
*
* Blank lines should remain in-tact, as should:
* - Indented
* - Things
*
* Some docs have:
* - List items that are not indented
*
* Sometimes after a blank line we'll have a note.
*
* *Note* that something.
*/
export interface A {
a: a;
}
''';
final output = parseString(input);
final input = {
"structures": [
{
"name": "A",
"properties": [],
"documentation": r"""
Describes the what this class in lots of words that wrap onto multiple lines that will need re-wrapping to format nicely when converted into Dart.
Blank lines should remain in-tact, as should:
- Indented
- Things
Some docs have:
- List items that are not indented
Sometimes after a blank line we'll have a note.
*Note* that something.""",
},
],
};
final output = readModel(input);
final interface = output[0] as Interface;
expect(interface.commentText, equals('''
Describes the what this class in lots of words that wrap onto multiple lines that will need re-wrapping to format nicely when converted into Dart.
@@ -232,11 +268,19 @@ Sometimes after a blank line we'll have a note.
*Note* that something.'''));
});
test('parses a type alias', () {
final input = '''
export type DocumentSelector = DocumentFilter[];
''';
final output = parseString(input);
test('reads a type alias', () {
final input = {
"typeAliases": [
{
"name": "DocumentSelector",
"type": {
"kind": "array",
"element": {"kind": "reference", "name": "DocumentFilter"}
},
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<TypeAlias>());
final typeAlias = output[0] as TypeAlias;
@@ -244,23 +288,54 @@ export type DocumentSelector = DocumentFilter[];
expect(typeAlias.baseType, isArrayOf(isSimpleType('DocumentFilter')));
});
test('parses a type alias that is a union of unnamed types', () {
final input = '''
export type NameOrLength = { name: string } | { length: number };
''';
final output = parseString(input);
test('reads a type alias that is a union of unnamed types', () {
final input = {
"typeAliases": [
{
"name": "NameOrLength",
"type": {
"kind": "or",
"items": [
{
"kind": "literal",
"value": {
"properties": [
{
"name": "name",
"type": {"kind": "base", "name": "string"}
},
]
},
},
{
"kind": "literal",
"value": {
"properties": [
{
"name": "length",
"type": {"kind": "base", "name": "number"}
},
]
},
},
]
},
},
],
};
final output = readModel(input);
expect(output, hasLength(3));
// Results should be the two inline interfaces followed by the type alias.
expect(output[0], const TypeMatcher<InlineInterface>());
final interface1 = output[0] as InlineInterface;
expect(output[0], const TypeMatcher<Interface>());
final interface1 = output[0] as Interface;
expect(interface1.name, equals('NameOrLength1'));
expect(interface1.members, hasLength(1));
expect(interface1.members[0].name, equals('name'));
expect(output[1], const TypeMatcher<InlineInterface>());
final interface2 = output[1] as InlineInterface;
expect(output[1], const TypeMatcher<Interface>());
final interface2 = output[1] as Interface;
expect(interface2.name, equals('NameOrLength2'));
expect(interface2.members, hasLength(1));
expect(interface2.members[0].name, equals('length'));
@@ -277,27 +352,37 @@ export type NameOrLength = { name: string } | { length: number };
expect(union.types[1], isSimpleType(interface2.name));
});
test('parses a namespace of constants', () {
final input = '''
export namespace ResourceOperationKind {
/**
* Supports creating new files and folders.
*/
export const Create: ResourceOperationKind = 'create';
/**
* Supports deleting existing files and folders.
*/
export const Delete: ResourceOperationKind = 'delete';
/**
* Supports renaming existing files and folders.
*/
export const Rename: ResourceOperationKind = 'rename';
}
''';
final output = parseString(input);
test('reads a namespace of constants', () {
final input = {
"enumerations": [
{
"name": "ResourceOperationKind",
"type": {"kind": "base", "name": "string"},
"values": [
{
"name": "Create",
"value": "create",
"documentation": "Supports creating new files and folders.",
},
{
"name": "Delete",
"value": "delete",
"documentation":
"Supports deleting existing files and folders.",
},
{
"name": "Rename",
"value": "rename",
"documentation":
"Supports renaming existing files and folders.",
},
],
},
]
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Namespace>());
final namespace = output[0] as Namespace;
expect(namespace.members, hasLength(3));
@@ -321,31 +406,33 @@ export namespace ResourceOperationKind {
equals('Supports deleting existing files and folders.'));
});
test('parses an enum using keywords as identifiers', () {
final input = '''
enum Foo {
namespace = 'namespace',
class = 'class',
enum = 'enum',
}
''';
final output = parseString(input);
expect(output, hasLength(1));
expect(output.first, const TypeMatcher<Namespace>());
final enum_ = output.first as Namespace;
expect(enum_.members, hasLength(3));
expect(enum_.members[0].name, equals('class'));
expect(enum_.members[1].name, equals('enum'));
expect(enum_.members[2].name, equals('namespace'));
});
test('parses a tuple in an array', () {
final input = '''
interface SomeInformation {
label: string | [number, number];
}
''';
final output = parseString(input);
test('reads a tuple in an array', () {
final input = {
"structures": [
{
"name": "SomeInformation",
"properties": [
{
"name": "label",
"type": {
"kind": "or",
"items": [
{"kind": "base", "name": "string"},
{
"kind": "tuple",
"items": [
{"kind": "base", "name": "number"},
{"kind": "base", "name": "number"}
]
}
]
},
},
],
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
@@ -360,13 +447,27 @@ interface SomeInformation {
expect(union.types[1], isSimpleType('string'));
});
test('parses an union including Object into a single type', () {
final input = '''
interface SomeInformation {
label: string | object;
}
''';
final output = parseString(input);
test('reads an union including LSPObject into a single type', () {
final input = {
"structures": [
{
"name": "SomeInformation",
"properties": [
{
"name": "label",
"type": {
"kind": "or",
"items": [
{"kind": "base", "name": "string"},
{"kind": "base", "name": "LSPObject"},
]
},
},
],
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
@@ -374,29 +475,24 @@ interface SomeInformation {
final field = interface.members.first as Field;
expect(field, const TypeMatcher<Field>());
expect(field.name, equals('label'));
expect(field.type, isSimpleType('object'));
expect(field.type, isSimpleType('LSPObject'));
});
test('parses multiple single-line comments into a single token', () {
final input = '''
// This is line 1
// This is line 2
interface SomeInformation {
}
''';
final output = parseString(input);
expect(output, hasLength(1));
expect(output[0].commentNode!.token.lexeme, equals('''// This is line 1
// This is line 2'''));
});
test('parses literal string values', () {
final input = '''
export interface MyType {
kind: 'one';
}
''';
final output = parseString(input);
test('reads literal string values', () {
final input = {
"structures": [
{
"name": "MyType",
"properties": [
{
"name": "kind",
"type": {"kind": "stringLiteral", "value": "one"},
},
],
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
@@ -410,13 +506,27 @@ export interface MyType {
expect(field.type, isLiteralOf(isSimpleType('string'), "'one'"));
});
test('parses literal union values', () {
final input = '''
export interface MyType {
kind: 'one' | 'two';
}
''';
final output = parseString(input);
test('reads literal union values', () {
final input = {
"structures": [
{
"name": "MyType",
"properties": [
{
"name": "kind",
"type": {
"kind": "or",
"items": [
{"kind": "stringLiteral", "value": "one"},
{"kind": "stringLiteral", "value": "two"},
]
},
},
],
},
],
};
final output = readModel(input);
expect(output, hasLength(1));
expect(output[0], const TypeMatcher<Interface>());
final interface = output[0] as Interface;
@@ -435,3 +545,6 @@ export interface MyType {
});
});
}
List<AstNode> readModel(Map<String, dynamic> model) =>
LspMetaModelCleaner().cleanTypes(LspMetaModelReader().readMap(model).types);
@@ -66,62 +66,16 @@ void recordTypes(List<AstNode> types) {
_sortSubtypes();
}
/// Renames types that may have been generated with bad names.
Iterable<AstNode> renameTypes(List<AstNode> types) sync* {
const renames = {
// TODO(dantup): These entries can be removed after the
// the migration to JSON meta_model.
'ClientCapabilitiesWindow': 'WindowClientCapabilities',
'ClientCapabilitiesWorkspace': 'WorkspaceClientCapabilities',
'ClientCapabilitiesWorkspaceFileOperations':
'FileOperationClientCapabilities',
'ServerCapabilitiesWorkspaceFileOperations': 'FileOperationOptions',
'ClientCapabilitiesGeneral': 'GeneralClientCapabilities',
'CompletionClientCapabilitiesCompletionItemInsertTextModeSupport':
'CompletionItemInsertTextModeSupport',
'CompletionClientCapabilitiesCompletionItemResolveSupport':
'CompletionItemResolveSupport',
'CompletionClientCapabilitiesCompletionItemTagSupport':
'CompletionItemTagSupport',
'CodeActionClientCapabilitiesCodeActionLiteralSupportCodeActionKind':
'CodeActionLiteralSupportCodeActionKind',
// In JSON model this becomes a union of literals which we assign improved
// names to (to avoid numeric suffixes).
'DocumentFilter': 'TextDocumentFilterWithScheme',
'ClientCapabilitiesGeneralStaleRequestSupport':
'GeneralClientCapabilitiesStaleRequestSupport',
'SignatureHelpClientCapabilitiesSignatureInformationParameterInformation':
'SignatureInformationParameterInformation',
'CompletionListItemDefaultsEditRange': 'CompletionItemEditRange',
};
for (final type in types) {
if (type is Interface) {
final newName = renames[type.name];
if (newName != null) {
// Replace with renamed interface.
yield Interface(
type.commentNode,
Token.identifier(newName),
type.typeArgs,
type.baseTypes,
type.members,
);
// Plus a TypeAlias for the old name.
yield TypeAlias(
type.commentNode,
Token.identifier(type.name),
Type.identifier(newName),
);
continue;
}
}
yield type;
}
}
TypeBase resolveTypeAlias(TypeBase type, {bool resolveEnumClasses = false}) {
if (type is Type) {
if (resolveEnumClasses) {
// Enums are no longer recorded with TypeAliases (as they were in the
// Markdown/TS spec) so must be resolved explicitly to their base types.
final enum_ = _namespaces[type.name];
if (enum_ != null) {
return enum_.typeOfValues;
}
}
// The LSP spec contains type aliases for `integer` and `uinteger` that map
// into the `number` type, with comments stating they must be integers. To
// preserve the improved typing, do _not_ resolve them to the `number`
@@ -165,23 +119,27 @@ String _formatCode(String code) {
return code;
}
/// Recursively gets all members from superclasses.
List<Field> _getAllFields(Interface? interface) {
/// Recursively gets all members from superclasses and returns them sorted
/// alphabetically.
List<Field> _getAllFields(Interface? interface) =>
_getSortedUnique(_getAllFieldsMap(interface).values.toList());
/// Recursively gets all members from superclasses keyed by field name.
Map<String, Field> _getAllFieldsMap(Interface? interface) {
// Handle missing interfaces (such as special cased interfaces that won't
// be included in this model).
if (interface == null) {
return [];
return {};
}
final allFields = interface.members
.whereType<Field>()
.followedBy(interface.baseTypes
// This cast is safe because base types are always real types.
.map((type) => _getAllFields(_interfaces[(type as Type).name]))
.expand((ts) => ts))
.toList();
return _getSortedUnique(allFields);
// It's possible our interface redefines something in a base type (for example
// where the base has `String` but this type overrides it with a literal such
// as `ResourceOperation`) so use a map to keep the most-specific by name.
return {
for (final baseType in interface.baseTypes)
..._getAllFieldsMap(_interfaces[baseType.name]),
for (final field in interface.members.whereType<Field>()) field.name: field,
};
}
/// Returns a copy of the list sorted by name with duplicates (by name+type) removed.
@@ -213,9 +171,9 @@ String _getTypeCheckFailureMessage(TypeBase type) {
type = resolveTypeAlias(type);
if (type is LiteralType) {
return 'must be the literal ${type.literal}';
return 'must be the literal ${type.valueAsLiteral}';
} else if (type is LiteralUnionType) {
return 'must be one of the literals ${type.literalTypes.map((t) => t.literal).join(', ')}';
return 'must be one of the literals ${type.literalTypes.map((t) => t.valueAsLiteral).join(', ')}';
} else {
return 'must be of type ${type.dartTypeWithTypeArgs}';
}
@@ -223,7 +181,7 @@ String _getTypeCheckFailureMessage(TypeBase type) {
bool _isOverride(Interface interface, Field field) {
for (var parentType in interface.baseTypes) {
var parent = _interfaces[(parentType as Type).name];
var parent = _interfaces[parentType.name];
if (parent != null) {
if (parent.members.any((m) => m.name == field.name)) {
return true;
@@ -244,6 +202,7 @@ bool _isSimpleType(TypeBase type) {
bool _isSpecType(TypeBase type) {
type = resolveTypeAlias(type);
return type is Type &&
!isAnyType(type) &&
(_interfaces.containsKey(type.name) ||
(_namespaces.containsKey(type.name)));
}
@@ -257,6 +216,7 @@ String _makeValidIdentifier(String identifier) {
'String': 'Str',
'class': 'class_',
'enum': 'enum_',
'null': 'null_',
};
return map[identifier] ?? identifier;
}
@@ -312,7 +272,7 @@ void _sortSubtypes() {
/// for enums.
String _specJsonType(TypeBase type) {
if (type is Type && _namespaces.containsKey(type.name)) {
final valueType = _namespaces[type.name]!.members.cast<Const>().first.type;
final valueType = _namespaces[type.name]!.typeOfValues;
return resolveTypeAlias(valueType, resolveEnumClasses: true)
.dartTypeWithTypeArgs;
}
@@ -432,11 +392,13 @@ void _writeConstructor(IndentableStringBuffer buffer, Interface interface) {
..writeIndented('${interface.name}({')
..write(allFields.map((field) {
final isLiteral = field.type is LiteralType;
final isRequired =
!isLiteral && !field.allowsNull && !field.allowsUndefined;
final isRequired = !isLiteral &&
!field.allowsNull &&
!field.allowsUndefined &&
!isAnyType(field.type);
final requiredKeyword = isRequired ? 'required' : '';
final valueCode =
isLiteral ? ' = ${(field.type as LiteralType).literal}' : '';
isLiteral ? ' = ${(field.type as LiteralType).valueAsLiteral}' : '';
return '$requiredKeyword this.${field.name}$valueCode, ';
}).join())
..write('})');
@@ -450,10 +412,10 @@ void _writeConstructor(IndentableStringBuffer buffer, Interface interface) {
final type = field.type;
if (type is LiteralType) {
buffer
..writeIndentedln('if (${field.name} != ${type.literal}) {')
..writeIndentedln('if (${field.name} != ${type.valueAsLiteral}) {')
..indent()
..writeIndentedln(
"throw '${field.name} may only be the literal ${type.literal.replaceAll("'", "\\'")}';")
"throw '${field.name} may only be the literal ${type.valueAsLiteral.replaceAll("'", "\\'")}';")
..outdent()
..writeIndentedln('}');
}
@@ -492,16 +454,10 @@ void _writeDocCommentsAndAnnotations(
void _writeEnumClass(IndentableStringBuffer buffer, Namespace namespace) {
_writeDocCommentsAndAnnotations(buffer, namespace);
final consts = namespace.members.cast<Const>().toList();
final allowsAnyValue = enumClassAllowsAnyValue(namespace.name);
final constructorName = allowsAnyValue ? '' : '._';
final firstValueType = consts.first.type;
// Enums can have constant values in their fields so if a field is a literal
// use its underlying type for type checking.
final requiredValueType =
firstValueType is LiteralType ? firstValueType.type : firstValueType;
final typeOfValues =
resolveTypeAlias(requiredValueType, resolveEnumClasses: true);
final namespaceName = namespace.name;
final typeOfValues = namespace.typeOfValues;
final allowsAnyValue = enumClassAllowsAnyValue(namespaceName);
final constructorName = allowsAnyValue ? '' : '._';
buffer
..writeln('class $namespaceName implements ToJsonable {')
@@ -542,8 +498,10 @@ void _writeEnumClass(IndentableStringBuffer buffer, Namespace namespace) {
return;
}
_writeDocCommentsAndAnnotations(buffer, cons);
final memberName = _makeValidIdentifier(cons.name);
final value = cons.valueAsLiteral;
buffer.writeIndentedln(
'static const ${_makeValidIdentifier(cons.name)} = $namespaceName$constructorName(${cons.valueAsLiteral});');
'static const $memberName = $namespaceName$constructorName($value);');
});
buffer
..writeln()
@@ -683,7 +641,7 @@ void _writeFromJsonCodeForLiteralUnion(
{required bool allowsNull}) {
final allowedValues = [
if (allowsNull) null,
...union.literalTypes.map((t) => t.literal)
...union.literalTypes.map((t) => t.valueAsLiteral)
];
final valueType = union.literalTypes.first.dartTypeWithTypeArgs;
final cast = ' as $valueType${allowsNull ? '?' : ''}';
@@ -769,7 +727,7 @@ void _writeFromJsonConstructor(
// Add a local variable to allow type promotion (and avoid multiple lookups).
final localName = _makeValidIdentifier(field.name);
final localNameJson = '${localName}Json';
buffer.writeIndented("final $localNameJson = json['${field.name}'];");
buffer.writeIndentedln("final $localNameJson = json['${field.name}'];");
buffer.writeIndented('final $localName = ');
_writeFromJsonCode(buffer, field.type, localNameJson,
allowsNull: field.allowsNull || field.allowsUndefined);
@@ -824,6 +782,7 @@ void _writeHashCode(IndentableStringBuffer buffer, Interface interface) {
}
void _writeInterface(IndentableStringBuffer buffer, Interface interface) {
final isPrivate = interface.name.startsWith('_');
_writeDocCommentsAndAnnotations(buffer, interface);
buffer.writeIndented('class ${interface.nameWithTypeArgs} ');
@@ -836,7 +795,9 @@ void _writeInterface(IndentableStringBuffer buffer, Interface interface) {
buffer
..writeln('{')
..indent();
_writeJsonHandler(buffer, interface);
if (!isPrivate) {
_writeJsonHandler(buffer, interface);
}
_writeConstructor(buffer, interface);
_writeFromJsonConstructor(buffer, interface);
// Handle Consts and Fields separately, since we need to include superclass
@@ -1019,7 +980,7 @@ void _writeTypeCheckCondition(IndentableStringBuffer buffer,
buffer.write('$valueCode is$operator $fullDartType');
} else if (type is LiteralType) {
final equals = negation ? '!=' : '==';
buffer.write('$valueCode $equals ${type.literal}');
buffer.write('$valueCode $equals ${type.valueAsLiteral}');
} else if (_isSpecType(type)) {
buffer.write('$operator$dartType.canParse($valueCode, $reporter)');
} else if (type is ArrayType) {
@@ -4,14 +4,11 @@
import 'dart:io';
import 'package:analysis_server/src/utilities/strings.dart';
import 'package:args/args.dart';
import 'package:http/http.dart' as http;
import 'package:path/path.dart' as path;
import 'codegen_dart.dart';
import 'markdown.dart';
import 'typescript.dart';
import 'typescript_parser.dart';
Future<void> main(List<String> arguments) async {
@@ -28,14 +25,9 @@ Future<void> main(List<String> arguments) async {
final outFolder = path.join(packageFolder, 'lib', 'lsp_protocol');
Directory(outFolder).createSync();
// Collect definitions for types in the spec and our custom extensions.
var specTypes = await getSpecClasses(args);
var customTypes = getCustomClasses();
// Handle some renames of types where we generate names that might not be
// ideal.
specTypes = renameTypes(specTypes).toList();
customTypes = renameTypes(customTypes).toList();
// Collect definitions for types in the model and our custom extensions.
final specTypes = await getSpecClasses(args);
final customTypes = getCustomClasses();
// Record both sets of types in dictionaries for faster lookups, but also so
// they can reference each other and we can find the definitions during
@@ -65,96 +57,35 @@ final argParser = ArgParser()
help:
'Download the latest version of the LSP spec before generating types');
final String localLicensePath = path.join(
path.dirname(Platform.script.toFilePath()), 'lsp_meta_model.license.txt');
final String localSpecPath = path.join(
path.dirname(Platform.script.toFilePath()), 'lsp_specification.md');
path.dirname(Platform.script.toFilePath()), 'lsp_meta_model.json');
final Uri specLicenseUri = Uri.parse(
'https://raw.githubusercontent.com/Microsoft/language-server-protocol/gh-pages/License.txt');
'https://microsoft.github.io/language-server-protocol/License.txt');
/// The URI of the version of the spec to generate from. This should be periodically updated as
/// there's no longer a stable URI for the latest published version.
/// The URI of the version of the LSP meta model to generate from. This should
/// be periodically updated to the latest version.
final Uri specUri = Uri.parse(
'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.17/specification.md');
/// Pattern to extract inline types from the `result: {xx, yy }` notes in the spec.
/// Doesn't parse past full stops as some of these have english sentences tagged on
/// the end that we don't want to parse.
final _resultsInlineTypesPattern = RegExp(r'''\* result:[^\.{}]*({[^\.`]*})''');
'https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/metaModel/metaModel.json');
Future<void> downloadSpec() async {
final specResp = await http.get(specUri);
final licenseResp = await http.get(specLicenseUri);
final text = [
'''
This is an unmodified copy of the Language Server Protocol Specification,
downloaded from $specUri. It is the version of the specification that was
used to generate a portion of the Dart code used to support the protocol.
To regenerate the generated code, run the script in
"analysis_server/tool/lsp_spec/generate_all.dart" with no arguments. To
download the latest version of the specification before regenerating the
code, run the same script with an argument of "--download".''',
licenseResp.body,
await _fetchIncludes(specResp.body, specUri),
];
await File(localSpecPath).writeAsString(text.join('\n\n---\n\n'));
}
assert(specResp.statusCode == 200);
assert(licenseResp.statusCode == 200);
Namespace extractMethodsEnum(String spec) {
Const toConstant(String value) {
final comment = Comment(
Token(TokenType.COMMENT, '''Constant for the '$value' method.'''));
// Generate a safe name for the member from the string. Those that start with
// $/ will have the prefix removed and all slashes should be replaced with
// underscores.
final safeMemberName = value.replaceAll(r'$/', '').replaceAll('/', '_');
return Const(
comment,
Token.identifier(safeMemberName),
Type.identifier('string'),
Token(TokenType.STRING, "'$value'"),
);
}
final comment = Comment(Token(TokenType.COMMENT,
'Valid LSP methods known at the time of code generation from the spec.'));
final methodConstants = extractMethodNames(spec).map(toConstant).toList();
return Namespace(comment, Token.identifier('Method'), methodConstants);
}
/// Extract inline types found directly in the `results:` sections of the spec
/// that are not declared with their own names elsewhere.
List<AstNode> extractResultsInlineTypes(String spec) {
InlineInterface toInterface(String typeDef) {
// The definition passed here will be a bare inline type, such as:
//
// { range: Range, placeholder: string }
//
// In order to parse this, we'll just format it as a type alias and then
// run it through the standard parsing code.
final typeAlias = 'type temp = ${typeDef.replaceAll(',', ';')};';
final parsed = parseString(typeAlias);
// Extract the InlineInterface that was created.
final interface =
parsed.firstWhere((t) => t is InlineInterface) as InlineInterface;
// Create a new name based on the fields.
var newName = interface.members.map((m) => capitalize(m.name)).join('And');
return InlineInterface(newName, interface.members);
}
return _resultsInlineTypesPattern
.allMatches(spec)
.map((m) => m.group(1)!.trim())
.toList()
.map(toInterface)
.toList();
await File(localSpecPath).writeAsString(specResp.body);
await File(localLicensePath).writeAsString(
'This license is for the ${path.basename(localSpecPath)} file.\n\n'
'${path.basename(localLicensePath)} downloaded from: $specLicenseUri\n'
'${path.basename(localSpecPath)} downloaded from: $specUri\n'
'\n--\n\n'
'${licenseResp.body}',
);
}
String generatedFileHeader(int year, {bool importCustom = false}) => '''
@@ -179,6 +110,7 @@ const jsonEncoder = JsonEncoder.withIndent(' ');
''';
List<AstNode> getCustomClasses() {
/// Helper to create an interface type.
Interface interface(String name, List<Member> fields, {String? baseType}) {
return Interface(
null,
@@ -189,6 +121,7 @@ List<AstNode> getCustomClasses() {
);
}
/// Helper to create a field.
Field field(
String name, {
String? comment,
@@ -221,6 +154,26 @@ List<AstNode> getCustomClasses() {
Token.identifier('LSPObject'),
Type.Any,
),
// The DocumentFilter more complex in v3.17's meta_model (to allow
// TextDocumentFilters to be guaranteed to have at least one of language,
// pattern, scheme) but we only ever use a single type in the server so
// for compatibility, alias that type to the original TS-spec name.
// TODO(dantup): Improve this after the TS->JSON Spec migration.
TypeAlias(
null,
Token.identifier('DocumentFilter'),
Type.identifier('TextDocumentFilter2'),
),
// Similarly, the meta_model includes String as an option for
// DocumentSelector which is deprecated and we never previously supported
// (because the TypeScript spec did not include it in the type) so preserve
// that.
// TODO(dantup): Improve this after the TS->JSON Spec migration.
TypeAlias(
null,
Token.identifier('DocumentSelector'),
ArrayType(Type.identifier('TextDocumentFilterWithScheme')),
),
interface('Message', [
field('jsonrpc', type: 'string'),
field('clientRequestTime', type: 'int', canBeUndefined: true),
@@ -400,79 +353,10 @@ Future<List<AstNode>> getSpecClasses(ArgResults args) async {
if (download) {
await downloadSpec();
}
final spec = await readSpec();
final types = extractTypeScriptBlocks(spec)
.where(shouldIncludeScriptBlock)
.map(parseString)
.expand((f) => f)
.where(includeTypeDefinitionInOutput)
.toList();
final file = File(localSpecPath);
var model = LspMetaModelReader().readFile(file);
model = LspMetaModelCleaner().cleanModel(model);
// Generate an enum for all of the request methods to avoid strings.
types.add(extractMethodsEnum(spec));
// Extract additional inline types that are specified online in the `results`
// section of the doc.
types.addAll(extractResultsInlineTypes(spec));
return types;
}
Future<String> readSpec() => File(localSpecPath).readAsString();
/// Returns whether a script block should be parsed or not.
bool shouldIncludeScriptBlock(String input) {
// Skip over some typescript blocks that are known sample code and not part
// of the LSP spec.
if (input.trim() == r"export const EOL: string[] = ['\n', '\r\n', '\r'];" ||
input.startsWith('textDocument.codeAction.resolveSupport =') ||
input.startsWith('textDocument.inlayHint.resolveSupport =') ||
// These two are example definitions, the real definitions start "export"
// and contain some base classes.
input.startsWith('interface HoverParams {') ||
input.startsWith('interface HoverResult {')) {
return false;
}
// There are some code blocks that just have example JSON in them.
if (input.startsWith('{') && input.endsWith('}')) {
return false;
}
// There are some example blocks that just contain arrays with no definitions.
// They're most easily noted by ending with `]` which no valid TypeScript blocks
// do.
if (input.trim().endsWith(']')) {
return false;
}
// There's a chunk of typescript that is just a partial snippet from a real
// interface declared elsewhere that we can only detect by the leading comment.
if (input
.replaceAll('\r', '')
.startsWith('/**\n\t * Window specific client capabilities.')) {
return false;
}
return true;
}
/// Fetches and in-lines any includes that appear in [spec] in the form
/// `{% include_relative types/uri.md %}`.
Future<String> _fetchIncludes(String spec, Uri baseUri) async {
final pattern = RegExp(r'{% include_relative ([\w\-.\/]+.md) %}');
final includeStrings = <String, String>{};
for (final match in pattern.allMatches(spec)) {
final relativeUri = match.group(1)!;
final fullUri = baseUri.resolve(relativeUri);
final response = await http.get(fullUri);
if (response.statusCode != 200) {
throw 'Failed to fetch $fullUri (${response.statusCode} ${response.reasonPhrase})';
}
includeStrings[relativeUri] = response.body;
}
return spec.replaceAllMapped(
pattern,
(match) => includeStrings[match.group(1)!]!,
);
return model.types;
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,15 @@
This license is for the lsp_meta_model.json file.
lsp_meta_model.license.txt downloaded from: https://microsoft.github.io/language-server-protocol/License.txt
lsp_meta_model.json downloaded from: https://raw.githubusercontent.com/microsoft/vscode-languageserver-node/main/protocol/metaModel.json
--
Copyright (c) Microsoft Corporation.
All rights reserved.
Distributed under the following terms:
1. Documentation is licensed under the Creative Commons Attribution 3.0 United States License. Code is licensed under the MIT License.
2. This license does not grant you rights to use any trademarks or logos of Microsoft. For Microsofts general trademark guidelines, go to http://go.microsoft.com/fwlink/?LinkID=254653
@@ -0,0 +1,352 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'typescript_parser.dart';
/// Helper methods to clean the meta model to produce better Dart classes.
///
/// Cleaning includes:
///
/// - Unwrapping comments that have been wrapped in the source model
/// - Removing relative hyperlinks from comments that assume rendering in an
/// HTML page with anchors
/// - Merging types that are distinct in the meta model but we want as one
/// - Removing types in the spec that we will never use
/// - Renaming types that may have long or sub-optimal generated names
/// - Simplifying union types that contain duplicates/overlaps
class LspMetaModelCleaner {
/// A pattern to match newlines in source comments that are likely for
/// wrapping and not formatting. This allows us to rewrap based on our indent
/// level/line length without potentially introducing very short lines.
final _sourceCommentWrappingNewlinesPattern =
RegExp(r'[\w`\]\).]\n[\w`\[\(]');
final _sourceCommentDocumentLinksPattern =
RegExp(r'\[([`\w \-.]+)\] ?\((#[^)]+)\)');
/// Cleans an entire [LspMetaModel].
LspMetaModel cleanModel(LspMetaModel model) {
final types = cleanTypes(model.types);
return LspMetaModel(types);
}
/// Cleans a List of types.
List<AstNode> cleanTypes(List<AstNode> types) {
types = _mergeTypes(types);
types = types
.where((type) => _includeTypeInOutput(type.name))
.map(_clean)
.toList();
types = _renameTypes(types).toList();
return types;
}
/// Whether a type should be retained type signatures in generated code.
bool _allowTypeInUnions(TypeBase type) {
// Don't allow arrays of MarkedStrings, but do allow simple MarkedStrings.
// The only place that uses these are Hovers and we only send one value
// (to match the MarkupString equiv) so the array just makes the types
// unnecessarily complicated.
if (type is ArrayType) {
// TODO(dantup): Consider removing this, it's not adding much.
final elementType = type.elementType;
if (elementType is Type && elementType.name == 'MarkedString') {
return false;
}
}
return true;
}
/// Cleans a single [AstNode].
AstNode _clean(AstNode type) {
if (type is Interface) {
return _cleanInterface(type);
} else if (type is Namespace) {
return _cleanNamespace(type);
} else if (type is TypeAlias) {
return _cleanTypeAlias(type);
} else {
throw 'Cleaning $type is not implemented.';
}
}
Comment? _cleanComment(Comment? comment) {
if (comment == null) {
return comment;
}
var text = comment.text;
// Unwrap any wrapping in the source by replacing any matching newlines with
// spaces.
text = text.replaceAllMapped(
_sourceCommentWrappingNewlinesPattern,
(match) => match.group(0)!.replaceAll('\n', ' '),
);
// Strip any relative links that are intended for displaying online in the
// HTML spec.
text = text.replaceAllMapped(
_sourceCommentDocumentLinksPattern,
(match) => match.group(1)!,
);
return Comment(Token(TokenType.COMMENT, text));
}
Const _cleanConst(Const const_) {
return Const(
_cleanComment(const_.commentNode),
const_.nameToken,
_cleanType(const_.type),
const_.valueToken,
);
}
Field _cleanField(Field field) {
return Field(
_cleanComment(field.commentNode),
field.nameToken,
_cleanType(field.type),
allowsNull: field.allowsNull,
allowsUndefined: field.allowsUndefined,
);
}
Interface _cleanInterface(Interface interface) {
return Interface(
_cleanComment(interface.commentNode),
interface.nameToken,
interface.typeArgs,
interface.baseTypes
.where((type) => _includeTypeInOutput(type.name))
.toList(),
interface.members.map(_cleanMember).toList(),
);
}
Member _cleanMember(Member member) {
if (member is Field) {
return _cleanField(member);
} else if (member is Const) {
return _cleanConst(member);
} else {
throw 'Cleaning $member is not implemented.';
}
}
Namespace _cleanNamespace(Namespace namespace) {
return Namespace(
_cleanComment(namespace.commentNode),
namespace.nameToken,
namespace.typeOfValues,
namespace.members.map(_cleanMember).toList(),
);
}
TypeBase _cleanType(TypeBase type) {
if (type is UnionType) {
return _cleanUnionType(type);
} else if (type is ArrayType) {
return ArrayType(_cleanType(type.elementType));
} else {
return type;
}
}
TypeAlias _cleanTypeAlias(TypeAlias typeAlias) {
return TypeAlias(
_cleanComment(typeAlias.commentNode),
typeAlias.nameToken,
typeAlias.baseType,
);
}
/// Removes any duplicate types in a union.
///
/// For example, if we map multiple types into `Object?` we don't want to end
/// up with `Either2<Object?, Object?>`.
///
/// Key on `dartType` to ensure we combine different types that will map down
/// to the same type.
TypeBase _cleanUnionType(UnionType type) {
var uniqueTypes = Map.fromEntries(
type.types
.where(_allowTypeInUnions)
.map((t) => MapEntry(t.uniqueTypeIdentifier, t)),
).values.toList();
// If our list includes something that maps to Object? as well as other
// types, we should just treat the whole thing as Object? as we get no value
// typing Either4<bool, String, num, Object?> but it becomes much more
// difficult to use.
if (uniqueTypes.any(isAnyType)) {
return uniqueTypes.firstWhere(isAnyType);
}
// Finally, sort the types by name so that we always generate the same type
// for the same combination to improve reuse of helper methods used in
// multiple handlers.
uniqueTypes.sort(
(t1, t2) => t1.dartTypeWithTypeArgs.compareTo(t2.dartTypeWithTypeArgs));
// Recursively clean the inner types.
uniqueTypes = uniqueTypes.map(_cleanType).toList();
return uniqueTypes.length == 1
? uniqueTypes.single
: uniqueTypes.every(isLiteralType)
? LiteralUnionType(uniqueTypes.cast<LiteralType>())
: UnionType(uniqueTypes);
}
/// Some types are merged together. This method returns the type that [name]s
/// members should be merged into.
String? _getMergeTarget(String name) {
switch (name) {
// The meta model defines both `LSPErrorCodes` and `ErrorCodes`. The
// intention was that one is JSONRPC and one is LSP codes, but some codes
// were defined in the wrong enum with the wrong values, but kept for
// backwards compatibility. For simplicity, we merge them all into `ErrorCodes`.
case 'LSPErrorCodes':
return 'ErrorCodes';
// In the model, `InitializeParams` is defined as by two classes,
// `_InitializeParams` and `WorkspaceFoldersInitializeParams`. This
// split doesn't add anything but makes the types less clear so we
// merge them into `InitializeParams`.
case '_InitializeParams':
return 'InitializeParams';
case 'WorkspaceFoldersInitializeParams':
return 'InitializeParams';
default:
return null;
}
}
/// Removes types that are in the spec that we don't want to emit.
bool _includeTypeInOutput(String name) {
const ignoredTypes = {
// InitializeError is not used for v3.0 (Feb 2017) and by dropping it we don't
// have to handle any cases where both a namespace and interfaces are declared
// with the same name.
'InitializeError',
// We don't use `InitializeErrorCodes` as it contains only one error code
// that has been deprecated and we've never used.
'InitializeErrorCodes',
// Handled in custom classes now in preperation for JSON meta model which
// does not specify them.
'Message',
'RequestMessage',
'NotificationMessage',
'ResponseMessage',
'ResponseError',
// Merged into InitializeParams.
'_InitializeParams',
'WorkspaceFoldersInitializeParams',
// We don't use these clases and they weren't in the TS version of the
// spec so continue to not generate them until required.
'DidChangeConfigurationRegistrationOptions',
// LSPAny/LSPObject are used by the LSP spec for unions of basic types.
// We map these onto Object? and don't use this type (and don't support
// unions with so many types).
'LSPAny',
'LSPObject',
// The meta model currently includes an unwanted type named 'T' that we
// don't want to create a class for.
// TODO(dantup): Remove this once it's gone from the JSON model.
'T',
};
const ignoredPrefixes = {
// We don't emit MarkedString because it gets mapped to a simple String
// when getting the .dartType for it.
'MarkedString'
};
final shouldIgnore = ignoredTypes.contains(name) ||
ignoredPrefixes.any((ignore) => name.startsWith(ignore));
return !shouldIgnore;
}
AstNode _merge(AstNode source, AstNode dest) {
if (source.runtimeType != dest.runtimeType) {
throw 'Cannot merge ${source.runtimeType} into ${dest.runtimeType}';
}
if (source is Namespace && dest is Namespace) {
return Namespace(
dest.commentNode ?? source.commentNode,
dest.nameToken,
dest.typeOfValues,
[...dest.members, ...source.members],
);
} else if (source is Interface && dest is Interface) {
return Interface(
dest.commentNode ?? source.commentNode,
dest.nameToken,
dest.typeArgs,
[...dest.baseTypes, ...source.baseTypes],
[...dest.members, ...source.members],
);
}
throw 'Merging ${source.runtimeType}s is not yet supported';
}
List<AstNode> _mergeTypes(List<AstNode> types) {
final typesByName = {
for (final type in types) type.name: type,
};
assert(types.length == typesByName.length);
final typeNames = typesByName.keys.toList();
for (final typeName in typeNames) {
final targetName = _getMergeTarget(typeName);
if (targetName != null) {
final type = typesByName[typeName]!;
final target = typesByName[targetName]!;
typesByName[targetName] = _merge(type, target);
typesByName.remove(typeName);
}
}
return typesByName.values.toList();
}
/// Renames types that may have been generated with bad (or long) names.
Iterable<AstNode> _renameTypes(List<AstNode> types) sync* {
const renames = <String, String>{
'CodeActionClientCapabilitiesCodeActionLiteralSupportCodeActionKind':
'CodeActionLiteralSupportCodeActionKind',
'CompletionClientCapabilitiesCompletionItemInsertTextModeSupport':
'CompletionItemInsertTextModeSupport',
'CompletionClientCapabilitiesCompletionItemTagSupport':
'CompletionItemTagSupport',
'CompletionClientCapabilitiesCompletionItemResolveSupport':
'CompletionItemResolveSupport',
'CompletionListItemDefaultsEditRange': 'CompletionItemEditRange',
'SignatureHelpClientCapabilitiesSignatureInformationParameterInformation':
'SignatureInformationParameterInformation',
'TextDocumentFilter2': 'TextDocumentFilterWithScheme',
'PrepareRenameResult1': 'PlaceholderAndRange',
};
for (final type in types) {
if (type is Interface) {
final newName = renames[type.name];
if (newName != null) {
// Replace with renamed interface.
yield Interface(
type.commentNode,
Token.identifier(newName),
type.typeArgs,
type.baseTypes,
type.members,
);
// Plus a TypeAlias for the old name.
yield TypeAlias(
type.commentNode,
Token.identifier(type.name),
Type.identifier(newName),
);
continue;
}
}
yield type;
}
}
}
@@ -0,0 +1,302 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:analysis_server/src/utilities/strings.dart';
import 'package:collection/collection.dart';
import 'typescript.dart';
import 'typescript_parser.dart';
/// Reads the LSP 'meta_model.json' file and returns its types.
class LspMetaModelReader {
final _types = <AstNode>[];
/// A set of names already used (or reserved) by types that have been read.
final Set<String> _typeNames = {};
/// Characters to strip from member names.
final _memberNameInvalidCharPattern = RegExp(r'\$_?');
/// Patterns to replace with '_' in member names.
final _memberNameSeparatorPattern = RegExp(r'/');
/// Gets all types that have been read from the model JSON.
List<AstNode> get types => _types.toList();
/// Creates a [Comment] from [text] if it is a valid string.
Comment? comment(dynamic text) =>
text is String ? Comment(Token(TokenType.COMMENT, text)) : null;
/// Reads all spec types from [file].
LspMetaModel readFile(File file) {
final modelJson = file.readAsStringSync();
final model = jsonDecode(modelJson) as Map<String, Object?>;
return readMap(model);
}
/// Reads all spec types from [model].
LspMetaModel readMap(Map<String, dynamic> model) {
final requests = model['requests'] as List?;
final notifications = model['notifications'] as List?;
final structures = model['structures'] as List?;
final enums = model['enumerations'] as List?;
final typeAliases = model['typeAliases'] as List?;
[
...?structures?.map(_readStructure),
...?enums?.map((e) => _readEnum(e)),
...?typeAliases?.map(_readTypeAlias),
].forEach(_addType);
final methodNames =
_createMethodNamesEnum([...?requests, ...?notifications]);
if (methodNames != null) {
_addType(methodNames);
}
return LspMetaModel(types);
}
/// Adds [type] to the current list and prevents its name from being used
/// by generated interfaces.
void _addType(AstNode type) {
_typeNames.add(type.name);
_types.add(type);
}
String _camelCase(String str) =>
str.substring(0, 1).toLowerCase() + str.substring(1);
/// Creates an enum for all LSP method names.
Namespace? _createMethodNamesEnum(List items) {
Const toConstant(String value) {
final comment = Comment(
Token(TokenType.COMMENT, '''Constant for the '$value' method.'''));
return Const(
comment,
Token.identifier(_generateMemberName(value, camelCase: true)),
Type.identifier('string'),
Token(TokenType.STRING, value),
);
}
final methodConstants =
items.map((item) => item['method'] as String).map(toConstant).toList();
if (methodConstants.isEmpty) {
return null;
}
final doc = comment('All standard LSP Methods read from the JSON spec.');
return Namespace(
doc,
Token.identifier('Method'),
Type.identifier('string'),
methodConstants,
);
}
Const _extractEnumValue(TypeBase parent, dynamic model) {
final name = model['name'] as String;
return Const(
comment(model['documentation']),
Token.identifier(_generateMemberName(name)),
parent,
Token(
model['value'] is int
? TokenType.NUMBER
: model['value'] is String
? TokenType.STRING
: throw 'Unknown enum value type $model',
model['value'].toString(),
),
);
}
Member _extractMember(String parentName, dynamic model) {
final name = model['name'] as String;
var type = _extractType(parentName, name, model['type']);
// Unions may contain `null` types which we promote up to the field.
var allowsNull = false;
if (type is UnionType) {
final types = type.types;
// Extract and strip `null`s from the union.
if (types.any(isNullType)) {
allowsNull = true;
type = UnionType(types.whereNot(isNullType).toList());
}
}
return Field(
comment(model['documentation']),
Token.identifier(_generateMemberName(name)),
type,
allowsNull: allowsNull,
allowsUndefined: model['optional'] == true,
);
}
/// Reads the type of [model].
TypeBase _extractType(String parentName, String? fieldName, dynamic model) {
final improvedType = getImprovedType(parentName, fieldName);
if (improvedType != null) {
return improvedType;
}
if (model['kind'] == 'reference' || model['kind'] == 'base') {
// Reference kinds are other named interfaces defined in the spec, base are
// other named types defined elsewhere.
return Type.identifier(model['name'] as String);
} else if (model['kind'] == 'array') {
return ArrayType(
_extractType(parentName, fieldName, model['element']!),
);
} else if (model['kind'] == 'map') {
final name = fieldName ?? '';
return MapType(
_extractType(parentName, '${name}Key', model['key']!),
_extractType(parentName, '${name}Value', model['value']!),
);
} else if (model['kind'] == 'literal') {
// "Literal" here means an inline/anonymous type.
final inlineTypeName = _generateTypeName(
parentName,
fieldName ?? '',
);
// First record the definition of the anonymous type itself.
final members = (model['value']['properties'] as List)
.map((p) => _extractMember(inlineTypeName, p))
.toList();
_addType(Interface.inline(inlineTypeName, members));
// Then return its name.
return Type.identifier(inlineTypeName);
} else if (model['kind'] == 'stringLiteral') {
return LiteralType(
Type.identifier('string'),
model['value'] as String,
);
} else if (model['kind'] == 'or') {
// Ensure the parent name is reserved so we don't try to reuse its name
// if we're parsing something without a field name.
_typeNames.add(parentName);
final itemTypes = model['items'] as List;
final types = itemTypes.map((item) {
final generatedName = _generateAvailableTypeName(parentName, fieldName);
return _extractType(generatedName, null, item);
}).toList();
return UnionType(types);
} else if (model['kind'] == 'tuple') {
// We currently just map tuples to an array of any of the types. The
// LSP 3.17 spec only has one tuple which is `[number, number]`.
final itemTypes = model['items'] as List;
final types = itemTypes.mapIndexed((index, item) {
final suffix = index + 1;
final name = fieldName ?? '';
final thisName = '$name$suffix';
return _extractType(parentName, thisName, item);
}).toList();
return ArrayType(UnionType(types));
} else {
throw 'Unable to extract type from $model';
}
}
/// Generates an available name for a node.
///
/// If the computed name is already used, a number will be appended to the
/// end.
String _generateAvailableTypeName(String containerName, String? fieldName) {
final name = _generateTypeName(containerName, fieldName ?? '');
final requiresSuffix = fieldName == null;
// If the name has already been taken, try appending a number and try
// again.
String generatedName;
var suffixIndex = 1;
do {
if (suffixIndex > 20) {
throw 'Failed to generate an available name for $name';
}
generatedName =
requiresSuffix || suffixIndex > 1 ? '$name$suffixIndex' : name;
suffixIndex++;
} while (_typeNames.contains(generatedName));
return generatedName;
}
/// Generates a valid name for a member.
String _generateMemberName(String name, {bool camelCase = false}) {
// Replace any seperators like `/` with `_`.
name = name.replaceAll(_memberNameSeparatorPattern, '_');
// Replace out any characters we don't want in member names.
name = name.replaceAll(_memberNameInvalidCharPattern, '');
// TODO(dantup): Remove this condition and always do camelCase in a future
// CL to reduce the migration diff.
if (camelCase) {
name = _camelCase(name);
}
return name;
}
/// Generates a valid name for a type.
String _generateTypeName(String parent, String child) {
// Some classes are private (`_InitializeParams`) but still exposed via
// other classes (`InitializeParams`) but the child types still need to be
// exposed, so remove any leading underscores.
if (parent.startsWith('_')) {
parent = parent.substring(1);
}
return '${capitalize(parent)}${capitalize(child)}';
}
Namespace _readEnum(dynamic model) {
final name = model['name'] as String;
final nameToken = Token.identifier(name);
final type = Type.identifier(name);
final baseType = _extractType(name, null, model['type']);
return Namespace(
comment(model['documentation']),
nameToken,
baseType,
[
...?(model['values'] as List?)?.map((p) => _extractEnumValue(type, p)),
],
);
}
AstNode _readStructure(dynamic model) {
final name = model['name'] as String;
return Interface(
comment(model['documentation']),
Token.identifier(name),
[],
[
...?(model['extends'] as List?)
?.map((e) => Type.identifier(e['name'] as String)),
...?(model['mixins'] as List?)
?.map((e) => Type.identifier(e['name'] as String)),
],
[
...?(model['properties'] as List?)?.map((p) => _extractMember(name, p)),
],
);
}
TypeAlias _readTypeAlias(dynamic model) {
final name = model['name'] as String;
return TypeAlias(
comment(model['documentation']),
Token.identifier(name),
_extractType(name, null, model['type']),
);
}
}
@@ -4,45 +4,6 @@
import 'typescript_parser.dart';
/// Removes types that are in the spec that we don't want in other signatures.
bool allowTypeInSignatures(TypeBase type) {
// Don't allow arrays of MarkedStrings, but do allow simple MarkedStrings.
// The only place that uses these are Hovers and we only send one value
// (to match the MarkupString equiv) so the array just makes the types
// unnecessarily complicated.
if (type is ArrayType) {
final elementType = type.elementType;
if (elementType is Type && elementType.name == 'MarkedString') {
return false;
}
}
return true;
}
String cleanComment(String comment) {
// Remove the start/end comment markers.
if (comment.startsWith('/**') && comment.endsWith('*/')) {
comment = comment.substring(3, comment.length - 2);
} else if (comment.startsWith('//')) {
comment = comment.substring(2);
}
final commentLinePrefixes = RegExp(r'\n\s*\* ?');
final nonConcurrentNewlines = RegExp(r'\n(?![\n\s\-*])');
final newLinesThatRequireReinserting = RegExp(r'\n (\w)');
// Remove any Windows newlines from the source.
comment = comment.replaceAll('\r', '');
// Remove the * prefixes.
comment = comment.replaceAll(commentLinePrefixes, '\n');
// Remove and newlines that look like wrapped text.
comment = comment.replaceAll(nonConcurrentNewlines, ' ');
// The above will remove one of the newlines when there are two, so we need
// to re-insert newlines for any block that starts immediately after a newline.
comment = comment.replaceAllMapped(
newLinesThatRequireReinserting, (m) => '\n\n${m.group(1)}');
return comment.trim();
}
/// Improves types in generated code, including:
///
/// - Fixes up some enum types that are not as specific as they could be in the
@@ -52,7 +13,7 @@ String cleanComment(String comment) {
/// - Narrows unions to single types where they're only generated on the server
/// and we know we always use a specific type. This avoids wrapping a lot
/// of code in `EitherX<Y,Z>.tX()` and simplifies the testing of them.
String? getImprovedType(String interfaceName, String? fieldName) {
TypeBase? getImprovedType(String interfaceName, String? fieldName) {
const improvedTypeMappings = <String, Map<String, String>>{
'Diagnostic': {
'severity': 'DiagnosticSeverity',
@@ -100,33 +61,15 @@ String? getImprovedType(String interfaceName, String? fieldName) {
final interface = improvedTypeMappings[interfaceName];
return interface != null ? interface[fieldName] : null;
}
final improvedTypeName = interface != null ? interface[fieldName] : null;
/// Removes types that are in the spec that we don't want to emit.
bool includeTypeDefinitionInOutput(AstNode node) {
const ignoredTypes = {
// InitializeError is not used for v3.0 (Feb 2017) and by dropping it we don't
// have to handle any cases where both a namespace and interfaces are declared
// with the same name.
'InitializeError',
// We don't use `InitializeErrorCodes` as it contains only one error code
// that has been deprecated and we've never used.
'InitializeErrorCodes',
// Handled in custom classes now in preperation for JSON meta model which
// does not specify them.
'Message',
'RequestMessage',
'NotificationMessage',
'ResponseMessage',
'ResponseError',
};
const ignoredPrefixes = {
// We don't emit MarkedString because it gets mapped to a simple String
// when getting the .dartType for it.
'MarkedString'
};
final shouldIgnore = ignoredTypes.contains(node.name) ||
ignoredPrefixes.any((ignore) => node.name.startsWith(ignore));
return !shouldIgnore;
return improvedTypeName != null
? improvedTypeName.endsWith('[]')
? ArrayType(Type.identifier(
improvedTypeName.substring(0, improvedTypeName.length - 2)))
: improvedTypeName.endsWith('?')
? UnionType.nullable(Type.identifier(
improvedTypeName.substring(0, improvedTypeName.length - 1)))
: Type.identifier(improvedTypeName)
: null;
}
@@ -2,31 +2,18 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:math';
import 'package:analysis_server/src/utilities/strings.dart' show capitalize;
import 'package:collection/collection.dart';
import 'codegen_dart.dart';
import 'typescript.dart';
export 'meta_model_cleaner.dart';
export 'meta_model_reader.dart';
/// A fabricated field name for indexers in case they result in generation
/// of type names for inline types.
const fieldNameForIndexer = 'indexer';
final _keywords = const <String, TokenType>{
'class': TokenType.CLASS_KEYWORD,
'const': TokenType.CONST_KEYWORD,
'enum': TokenType.ENUM_KEYWORD,
'export': TokenType.EXPORT_KEYWORD,
'extends': TokenType.EXTENDS_KEYWORD,
'interface': TokenType.INTERFACE_KEYWORD,
'namespace': TokenType.NAMESPACE_KEYWORD,
'readonly': TokenType.READONLY_KEYWORD,
};
final _validIdentifierCharacters = RegExp('[a-zA-Z0-9_]');
/// Whether this type allows any value (including null).
bool isAnyType(TypeBase t) =>
t is Type &&
(t.name == 'any' ||
@@ -40,13 +27,6 @@ bool isNullType(TypeBase t) => t is Type && t.name == 'null';
bool isUndefinedType(TypeBase t) => t is Type && t.name == 'undefined';
List<AstNode> parseString(String input) {
final scanner = Scanner(input);
final tokens = scanner.scan();
final parser = Parser(tokens);
return parser.parse();
}
TypeBase typeOfLiteral(Token token) {
final tokenType = token.type;
final typeName = tokenType == TokenType.STRING
@@ -83,14 +63,14 @@ class Comment extends AstNode {
final String text;
Comment(this.token)
: text = cleanComment(token.lexeme),
: text = token.lexeme,
super(null);
@override
String get name => throw UnsupportedError('Comments do not have a name.');
}
class Const extends Member {
class Const extends Member with LiteralValueMixin {
Token nameToken;
TypeBase type;
Token valueToken;
@@ -99,15 +79,7 @@ class Const extends Member {
@override
String get name => nameToken.lexeme;
String get valueAsLiteral {
var lexeme = valueToken.lexeme;
if (type.dartType == 'String' && lexeme.contains(r'$')) {
// lexeme already includes the quotes as read from the spec.
return 'r$lexeme';
} else {
return lexeme;
}
}
String get valueAsLiteral => _asLiteral(valueToken.lexeme);
}
class Field extends Member {
@@ -140,30 +112,10 @@ class FixedValueField extends Field {
allowsNull: allowsNull, allowsUndefined: allowsUndefined);
}
class Indexer extends Member {
final TypeBase indexType;
final TypeBase valueType;
Indexer(
super.comment,
this.indexType,
this.valueType,
);
@override
String get name => fieldNameForIndexer;
}
class InlineInterface extends Interface {
InlineInterface(
String name,
List<Member> members,
) : super(null, Token.identifier(name), [], [], members);
}
class Interface extends AstNode {
final Token nameToken;
final List<Token> typeArgs;
final List<TypeBase> baseTypes;
final List<Type> baseTypes;
final List<Member> members;
Interface(
@@ -177,6 +129,9 @@ class Interface extends AstNode {
members.sortBy((member) => member.name.toLowerCase());
}
Interface.inline(String name, List<Member> members)
: this(null, Token.identifier(name), [], [], members);
@override
String get name => nameToken.lexeme;
String get nameWithTypeArgs => '$name$typeArgsString';
@@ -186,11 +141,11 @@ class Interface extends AstNode {
: '';
}
class LiteralType extends TypeBase {
class LiteralType extends TypeBase with LiteralValueMixin {
final TypeBase type;
final String literal;
final String _literal;
LiteralType(this.type, this.literal);
LiteralType(this.type, this._literal);
@override
String get dartType => type.dartType;
@@ -199,7 +154,9 @@ class LiteralType extends TypeBase {
String get typeArgsString => type.typeArgsString;
@override
String get uniqueTypeIdentifier => '$literal:${super.uniqueTypeIdentifier}';
String get uniqueTypeIdentifier => '$_literal:${super.uniqueTypeIdentifier}';
String get valueAsLiteral => _asLiteral(_literal);
}
/// A special class of Union types where the values are all literals of the same
@@ -216,6 +173,24 @@ class LiteralUnionType extends UnionType {
String get typeArgsString => types.first.typeArgsString;
}
mixin LiteralValueMixin {
String _asLiteral(String value) {
if (num.tryParse(value) == null) {
// Add quotes around strings.
final prefix = value.contains(r'$') ? 'r' : '';
return "$prefix'$value'";
} else {
return value;
}
}
}
class LspMetaModel {
final List<AstNode> types;
LspMetaModel(this.types);
}
class MapType extends TypeBase {
final TypeBase indexType;
final TypeBase valueType;
@@ -236,10 +211,12 @@ abstract class Member extends AstNode {
class Namespace extends AstNode {
final Token nameToken;
final TypeBase typeOfValues;
final List<Member> members;
Namespace(
super.comment,
this.nameToken,
this.typeOfValues,
this.members,
) {
members.sortBy((member) => member.name.toLowerCase());
@@ -249,677 +226,6 @@ class Namespace extends AstNode {
String get name => nameToken.lexeme;
}
class Parser {
final List<Token> _tokens;
int _current = 0;
final List<AstNode> _nodes = [];
/// A set of names already used (or reserved) by nodes.
final Set<String> _nodeNames = {};
Parser(this._tokens);
bool get _isAtEnd => _peek().type == TokenType.EOF;
List<AstNode> parse() {
if (_nodes.isEmpty) {
while (!_isAtEnd) {
_addNode(_topLevel());
// Consume any trailing semicolons.
_match([TokenType.SEMI_COLON]);
}
}
return _nodes;
}
/// Adds [node] to the current list and prevents its name from being used
/// by generated interfaces.
void _addNode(AstNode node) {
_nodeNames.add(node.name);
_nodes.add(node);
}
/// Returns the current token and moves to the next.
Token _advance() => _tokenAt(_current++);
/// Checks if the next token is [type] without advancing.
bool _check(TokenType type) => !_isAtEnd && _peek().type == type;
Comment? _comment() {
if (_peek().type != TokenType.COMMENT) {
return null;
}
return Comment(_advance());
}
Const _const(String containerName, Comment? leadingComment) {
_eatUnwantedKeywords();
final name = _consume(TokenType.IDENTIFIER, 'Expected identifier');
TypeBase? type;
if (_match([TokenType.COLON])) {
type = _type(containerName, name.lexeme);
}
final value = _match([TokenType.EQUAL]) ? _advance() : null;
if (type == null && value != null) {
type = typeOfLiteral(value);
}
_consume(TokenType.SEMI_COLON, 'Expected ;');
return Const(leadingComment, name, type!, value!);
}
/// Ensures the next token is [type] and moves to the next, throwing [message]
/// if not.
Token _consume(TokenType type, String message) {
// Skip over any inline comments when looking for a specific token.
_match([TokenType.COMMENT]);
if (_check(type)) {
return _advance();
}
// The scanner currently reads keywords with specific token types
// (eg. TokenType.NAMESPACE_KEYWORD) however v3.16 of the LSP spec also uses
// some of these words as identifiers. If the requested type is an identifier
// but we have a keyword token, then treat it as an identifier.
if (type == TokenType.IDENTIFIER) {
final next = !_isAtEnd ? _peek() : null;
if (next != null && _isKeyword(next.type)) {
_advance();
return Token(TokenType.IDENTIFIER, next.lexeme);
}
}
throw '$message\n\n${_peek()}';
}
void _eatUnwantedKeywords() {
_match([TokenType.EXPORT_KEYWORD]);
_match([TokenType.READONLY_KEYWORD]);
}
Namespace _enum(Comment? leadingComment) {
final name = _consume(TokenType.IDENTIFIER, 'Expected identifier');
_consume(TokenType.LEFT_BRACE, 'Expected {');
final consts = <Const>[];
while (!_check(TokenType.RIGHT_BRACE)) {
consts.add(_enumValue(name.lexeme));
// Commas might not be present (eg. for last one).
_match([TokenType.COMMA]);
}
_consume(TokenType.RIGHT_BRACE, 'Expected }');
return Namespace(leadingComment, name, consts);
}
Const _enumValue(String enumName) {
final leadingComment = _comment();
final name = _consume(TokenType.IDENTIFIER, 'Expected identifier');
TypeBase? type;
if (_match([TokenType.COLON])) {
type = _type(enumName, name.lexeme);
}
final value = _match([TokenType.EQUAL]) ? _advance() : null;
if (type == null && value != null) {
type = typeOfLiteral(value);
}
return Const(leadingComment, name, type!, value!);
}
Field _field(String containerName, Comment? leadingComment) {
_eatUnwantedKeywords();
final name = _consume(TokenType.IDENTIFIER, 'Expected identifier');
var canBeUndefined = _match([TokenType.QUESTION]);
_consume(TokenType.COLON, 'Expected :');
TypeBase type;
Token? value;
type = _type(containerName, name.lexeme,
includeUndefined: canBeUndefined, improveTypes: true);
// Some fields have weird comments like this in the spec:
// {@link MessageType}
// These seem to be the correct type of the field, while the field is
// marked with number.
final commentText = leadingComment?.text;
if (commentText != null) {
final linkTypePattern = RegExp(r'See \{@link (\w+)\}\.?');
final linkTypeMatch = linkTypePattern.firstMatch(commentText);
if (linkTypeMatch != null) {
type = Type.identifier(linkTypeMatch.group(1)!);
leadingComment = Comment(Token(TokenType.COMMENT,
'// ${commentText.replaceAll(linkTypePattern, '')}'));
}
}
// Ideally this would be _consume(), but there are no semi-colons after the
// "inline types" since they're blocks.
_match([TokenType.SEMI_COLON]);
// Special handling for fields that have fixed values.
if (value != null) {
return FixedValueField(
leadingComment, name, value, type, false, canBeUndefined);
}
var canBeNull = false;
if (type is UnionType) {
// Since undefined and null can appear in the union type list but we want to
// handle it specially in the code generation, we promote them to fields on
// the Field.
canBeUndefined |= type.types.any(isUndefinedType);
canBeNull = type.types.any((t) => isNullType(t) || isAnyType(t));
// Finally, we need to remove them from the union.
final remainingTypes = type.types
.where((t) => !isNullType(t) && !isUndefinedType(t))
.toList();
// We also remove any types that are deprecated and/or we won't use to
// simplify the unions.
remainingTypes.removeWhere((t) => !allowTypeInSignatures(t));
type = _simplifyUnionTypes(remainingTypes);
} else if (isAnyType(type)) {
// There are values in the spec marked as `any` that allow nulls (for
// example, the result field on ResponseMessage can be null for a
// successful response that has no return value, eg. shutdown).
canBeNull = true;
}
return Field(leadingComment, name, type,
allowsNull: canBeNull, allowsUndefined: canBeUndefined);
}
/// Gets an available name for a node.
///
/// If the computed name is already used, a number will be appended to the
/// end.
String _getAvailableName(String containerName, String? fieldName) {
final name = _joinNames(containerName, fieldName ?? '');
final requiresSuffix = fieldName == null;
// If the name has already been taken, try appending a number and try
// again.
String generatedName;
var suffixIndex = 1;
do {
if (suffixIndex > 20) {
throw 'Failed to generate an available name for $name';
}
generatedName =
requiresSuffix || suffixIndex > 1 ? '$name$suffixIndex' : name;
suffixIndex++;
} while (_nodeNames.contains(generatedName));
return generatedName;
}
Indexer _indexer(String containerName, Comment? leadingComment) {
final indexer = _field(containerName, leadingComment);
_consume(TokenType.RIGHT_BRACKET, 'Expected ]');
_consume(TokenType.COLON, 'Expected :');
TypeBase type;
type = _type(containerName, fieldNameForIndexer, improveTypes: true);
//_consume(TokenType.RIGHT_BRACE, 'Expected }');
_match([TokenType.SEMI_COLON]);
return Indexer(leadingComment, indexer.type, type);
}
Interface _interface(Comment? leadingComment) {
final name = _consume(TokenType.IDENTIFIER, 'Expected identifier');
final typeArgs = <Token>[];
if (_match([TokenType.LESS])) {
while (true) {
typeArgs.add(_consume(TokenType.IDENTIFIER, 'Expected identifier'));
if (_check(TokenType.GREATER)) {
break;
}
_consume(TokenType.COMMA, 'Expected , or >');
}
_consume(TokenType.GREATER, 'Expected >');
}
final baseTypes = <TypeBase>[];
if (_match([TokenType.EXTENDS_KEYWORD])) {
while (true) {
baseTypes.add(_type(name.lexeme, null));
if (_check(TokenType.LEFT_BRACE)) {
break;
}
_consume(TokenType.COMMA, 'Expected , or {');
}
}
_consume(TokenType.LEFT_BRACE, 'Expected {');
final members = <Member>[];
while (!_check(TokenType.RIGHT_BRACE)) {
members.add(_member(name.lexeme));
}
_consume(TokenType.RIGHT_BRACE, 'Expected }');
return Interface(leadingComment, name, typeArgs, baseTypes, members);
}
bool _isKeyword(TokenType type) => _keywords.values.contains(type);
String _joinNames(String parent, String child) {
return '$parent${capitalize(child)}';
}
/// Returns [true] an advances if the next token is one of [types], otherwise
/// returns [false].
bool _match(List<TokenType> types) {
for (final type in types) {
if (_check(type)) {
_advance();
return true;
}
}
return false;
}
Member _member(String containerName) {
final leadingComment = _comment();
_eatUnwantedKeywords();
if (_match([TokenType.CONST_KEYWORD])) {
return _const(containerName, leadingComment);
} else if (_match([TokenType.LEFT_BRACKET])) {
return _indexer(containerName, leadingComment);
} else {
return _field(containerName, leadingComment);
}
}
Namespace _namespace(Comment? leadingComment) {
final name = _consume(TokenType.IDENTIFIER, 'Expected identifier');
_consume(TokenType.LEFT_BRACE, 'Expected {');
final members = <Member>[];
while (!_check(TokenType.RIGHT_BRACE)) {
members.add(_member(name.lexeme));
}
_consume(TokenType.RIGHT_BRACE, 'Expected }');
return Namespace(leadingComment, name, members);
}
/// Returns the next token without advancing.
Token _peek() => _tokenAt(_current);
/// Remove any duplicate types (for ex. if we map multiple types into Object?)
/// we don't want to end up with `Object? | Object?`. Key on dartType to
/// ensure we different types that will map down to the same type.
TypeBase _simplifyUnionTypes(List<TypeBase> types) {
final uniqueTypes = Map.fromEntries(
types.map((t) => MapEntry(t.uniqueTypeIdentifier, t)),
).values.toList();
// If our list includes something that maps to Object? as well as other
// types, we should just treat the whole thing as Object? as we get no value
// typing Either4<bool, String, num, Object?> but it becomes much more
// difficult to use.
if (uniqueTypes.any(isAnyType)) {
return uniqueTypes.firstWhere(isAnyType);
}
// Special case to simplify a complex type in the TypeScript spec that is
// hard to detect generically and is already simplified in the JSON model.
// The first type in the union is fully representable in the second and can
// be dropped.
// TODO(dantup): Remove this when switching to the JSON model.
if (uniqueTypes.length == 2 &&
uniqueTypes[0].dartTypeWithTypeArgs == 'List<TextDocumentEdit>' &&
uniqueTypes[1].dartTypeWithTypeArgs ==
'List<Either4<CreateFile, DeleteFile, RenameFile, TextDocumentEdit>>') {
return uniqueTypes[1];
}
return uniqueTypes.length == 1
? uniqueTypes.single
: uniqueTypes.every(isLiteralType)
? LiteralUnionType(uniqueTypes.cast<LiteralType>())
: UnionType(uniqueTypes);
}
Token _tokenAt(int index) =>
index < _tokens.length ? _tokens[index] : Token.EOF;
AstNode _topLevel() {
final leadingComment = _comment();
_match([TokenType.EXPORT_KEYWORD]);
final token = _peek();
if (_match([TokenType.NAMESPACE_KEYWORD])) {
return _namespace(leadingComment);
} else if (_match([TokenType.INTERFACE_KEYWORD])) {
return _interface(leadingComment);
} else if (_match([TokenType.CLASS_KEYWORD])) {
// Classes are the same as interfaces in this spec.
return _interface(leadingComment);
} else if (_match([TokenType.ENUM_KEYWORD])) {
return _enum(leadingComment);
} else if (token.type == TokenType.IDENTIFIER && token.lexeme == 'type') {
// TODO(dantup): This is a hack... We don't have a TYPE_KEYWORD because
// the spec has `type` as an identifier.
_advance(); // Eat the 'type' keyword.
return _typeAlias(leadingComment);
} else {
throw 'Unexpected token ${_peek()}';
}
}
TypeBase _type(
String containerName,
String? fieldName, {
bool includeUndefined = false,
bool improveTypes = false,
}) {
var types = <TypeBase>[];
if (includeUndefined) {
types.add(Type.Undefined);
}
while (true) {
TypeBase type;
if (_match([TokenType.LEFT_BRACE])) {
// Inline interfaces.
final generatedName = _getAvailableName(containerName, fieldName);
final members = <Member>[];
while (!_check(TokenType.RIGHT_BRACE)) {
members.add(_member(generatedName));
}
_consume(TokenType.RIGHT_BRACE, 'Expected }');
// Some of the inline interfaces have trailing commas (and some do not!)
_match([TokenType.COMMA]);
// If we have a single member that is an indexer type, we can use a Map.
if (members.length == 1 && members.single is Indexer) {
var indexer = members.single as Indexer;
type = MapType(indexer.indexType, indexer.valueType);
} else {
// Add a synthetic interface to the parsers list of nodes to represent this type.
_addNode(InlineInterface(generatedName, members));
// Record the type as a simple type that references this interface.
type = Type.identifier(generatedName);
}
} else if (_match([TokenType.LEFT_PAREN])) {
// Some types are in (parens), so we just parse the contents as a nested type.
type = _type(containerName, fieldName);
_consume(TokenType.RIGHT_PAREN, 'Expected )');
} else if (_check(TokenType.STRING) || _check(TokenType.NUMBER)) {
final token = _advance();
// In TS and the spec, literal values can be types:
// export const PlainText: 'plaintext' = 'plaintext';
// trace?: 'off' | 'messages' | 'verbose';
// export const Invoked: 1 = 1;
type = LiteralType(typeOfLiteral(token), token.lexeme);
} else if (_match([TokenType.LEFT_BRACKET])) {
// Tuples will just be converted to List/Array.
final tupleElementTypes = <TypeBase>[];
while (!_check(TokenType.RIGHT_BRACKET)) {
tupleElementTypes.add(_type(containerName, fieldName));
// Remove commas in between.
_match([TokenType.COMMA]);
}
_consume(TokenType.RIGHT_BRACKET, 'Expected ]');
var tupleType = _simplifyUnionTypes(tupleElementTypes);
type = ArrayType(tupleType);
} else {
var typeName = _consume(TokenType.IDENTIFIER, 'Expected identifier');
final typeArgs = <TypeBase>[];
if (_match([TokenType.LESS])) {
while (true) {
typeArgs.add(_type(containerName, fieldName));
if (_peek().type != TokenType.COMMA) {
_consume(TokenType.GREATER, 'Expected >');
break;
}
}
}
type = typeName.lexeme == 'Array'
? ArrayType(typeArgs.single)
: Type(typeName, typeArgs);
}
if (_match([TokenType.LEFT_BRACKET])) {
_consume(TokenType.RIGHT_BRACKET, 'Expected ]');
type = ArrayType(type);
}
// TODO(dantup): Handle types like This & That.
// For now, map to any.
if (_match([TokenType.AMPERSAND])) {
while (true) {
// Eat as many types/ampersands as we have.
_type(containerName, fieldName);
if (!_check(TokenType.AMPERSAND)) {
break;
}
}
type = Type.Any;
}
types.add(type);
if (!_match([TokenType.PIPE])) {
break;
}
}
var type = _simplifyUnionTypes(types);
// Handle improved type mappings for things that aren't very tight in the spec.
if (improveTypes) {
final improvedTypeName = getImprovedType(containerName, fieldName);
if (improvedTypeName != null) {
type = improvedTypeName.endsWith('[]')
? ArrayType(Type.identifier(
improvedTypeName.substring(0, improvedTypeName.length - 2)))
: Type.identifier(improvedTypeName);
}
}
return type;
}
TypeAlias _typeAlias(Comment? leadingComment) {
final name = _consume(TokenType.IDENTIFIER, 'Expected identifier');
_consume(TokenType.EQUAL, 'Expected =');
// Reserve the name for this alias before we start reading its type so that
// inline/literal types will not try to compute the same name if they do
// not have field names.
_nodeNames.add(name.lexeme);
final type = _type(name.lexeme, null);
if (!_isAtEnd) {
_consume(TokenType.SEMI_COLON, 'Expected ;');
}
return TypeAlias(leadingComment, name, type);
}
}
class Scanner {
final String _source;
int _startOfToken = 0;
int _currentPos = 0;
final _tokens = <Token>[];
Scanner(this._source);
bool get _isAtEnd => _currentPos >= _source.length;
bool get _isNextAtEnd => _currentPos + 1 >= _source.length;
List<Token> scan() {
while (!_isAtEnd) {
_startOfToken = _currentPos;
_scanToken();
}
return _tokens;
}
void _addToken(TokenType type, {bool mergeSameTypes = false}) {
var text = _source.substring(_startOfToken, _currentPos);
// Consecutive tokens of some types (for example Comments) are merged
// together.
if (mergeSameTypes && _tokens.isNotEmpty && type == _tokens.last.type) {
text = '${_tokens.last.lexeme}\n$text';
_tokens.removeLast();
}
_tokens.add(Token(type, text));
}
String _advance() => _currentPos < _source.length
? _source[_currentPos++]
: throw 'Cannot advance past end of source';
void _identifier() {
while (_isAlpha(_peek())) {
_advance();
}
final string = _source.substring(_startOfToken, _currentPos);
var keyword = _keywords[string];
if (keyword != null) {
_addToken(keyword);
} else {
_addToken(TokenType.IDENTIFIER);
}
}
bool _isAlpha(String? s) =>
s != null && _validIdentifierCharacters.hasMatch(s);
bool _isDigit(String? s) => s != null && (s.codeUnitAt(0) ^ 0x30) <= 9;
bool _match(String expected) {
if (_isAtEnd || _source[_currentPos] != expected) {
return false;
}
_currentPos++;
return true;
}
void _number() {
// Optionally process a negative.
_match('-');
while (_isDigit(_peek())) {
_advance();
}
// Handle fractional parts.
if (_peek() == '.' && _isDigit(_peekNext())) {
// Consume the decimal point.
_advance();
while (_isDigit(_peek())) {
_advance();
}
}
_addToken(TokenType.NUMBER);
}
String? _peek() => _isAtEnd ? null : _source[_currentPos];
String? _peekNext() => _isNextAtEnd ? null : _source[_currentPos + 1];
void _scanToken() {
const singleCharTokens = <String, TokenType>{
',': TokenType.COMMA,
';': TokenType.SEMI_COLON,
':': TokenType.COLON,
'?': TokenType.QUESTION,
'.': TokenType.DOT,
'(': TokenType.LEFT_PAREN,
')': TokenType.RIGHT_PAREN,
'[': TokenType.LEFT_BRACKET,
']': TokenType.RIGHT_BRACKET,
'{': TokenType.LEFT_BRACE,
'}': TokenType.RIGHT_BRACE,
'*': TokenType.STAR,
'&': TokenType.AMPERSAND,
'=': TokenType.EQUAL,
'|': TokenType.PIPE,
};
final c = _advance();
var token = singleCharTokens[c];
if (token != null) {
_addToken(token);
return;
}
switch (c) {
case '/':
if (_match('*')) {
// Block comment.
while (!_isAtEnd && (_peek() != '*' || _peekNext() != '/')) {
_advance();
}
// Eat the closing comment markers detected above.
if (!_isAtEnd) {
_advance();
_advance();
}
_addToken(TokenType.COMMENT, mergeSameTypes: true);
} else if (_match('/')) {
// Single line comment.
while (_peek() != '\n' && !_isAtEnd) {
_advance();
}
_addToken(TokenType.COMMENT, mergeSameTypes: true);
} else {
_addToken(TokenType.SLASH);
}
break;
case '<':
_addToken(_match('=') ? TokenType.LESS_EQUAL : TokenType.LESS);
break;
case '>':
_addToken(_match('=') ? TokenType.GREATER_EQUAL : TokenType.GREATER);
break;
case ' ':
case '\r':
case '\n':
case '\t':
// Whitespace.
break;
case '"':
case "'":
_string(c);
break;
default:
if (_isDigit(c) || c == '-' && _isDigit(_peek())) {
_number();
} else if (_isAlpha(c)) {
_identifier();
} else {
final start = max(0, _currentPos - 20);
final end = min(_currentPos + 20, _source.length);
final snippet = _source.substring(start, end);
throw "Unexpected character '$c'.\n\n$snippet";
}
break;
}
}
void _string(String terminator) {
// TODO(dantup): Handle escape sequences, inc. quotes.
while (!_isAtEnd && _peek() != terminator) {
_advance();
if (_isAtEnd) {
throw 'Unterminated string.';
}
}
// Skip over the closing terminator.
_advance();
_addToken(TokenType.STRING);
}
}
class Token {
static final Token EOF = Token(TokenType.EOF, '');
@@ -973,6 +279,7 @@ enum TokenType {
class Type extends TypeBase {
static final TypeBase Undefined = Type.identifier('undefined');
static final TypeBase Null_ = Type.identifier('null');
static final TypeBase Any = Type.identifier('any');
final Token nameToken;
final List<TypeBase> typeArgs;
@@ -998,9 +305,14 @@ class Type extends TypeBase {
'string': 'String',
'number': 'num',
'integer': 'int',
// Map decimal to num because clients may sent "1.0" or "1" and we want
// to consider both valid.
'decimal': 'num',
'uinteger': 'int',
'any': 'Object?',
'LSPAny': 'Object?',
'object': 'Object?',
'LSPObject': 'Object?',
// Simplify MarkedString from
// string | { language: string; value: string }
// to just String
@@ -1059,6 +371,8 @@ class UnionType extends TypeBase {
types.sortBy((type) => type.dartTypeWithTypeArgs.toLowerCase());
}
UnionType.nullable(TypeBase type) : this([type, Type.Null_]);
@override
String get dartType {
if (types.length > 4) {