From 57863dc64b2df54191f7459eb3434aa4a3858bc6 Mon Sep 17 00:00:00 2001 From: Alexander Markov Date: Fri, 23 Jun 2023 20:14:08 +0000 Subject: [PATCH] [vm/aot/tfa] Tree-shake unused libraries On a large app (which apparently has too many unused dependencies), size of the AOT kernel file: before: 464 MB after: 108 MB TEST=pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart Bug: b/287638965 Change-Id: I79a26305c00741babb6a69a18919983b398109e5 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/310772 Reviewed-by: Slava Egorov Commit-Queue: Alexander Markov Reviewed-by: Martin Kustermann --- .../type_flow/transformer.dart | 81 +++++++++++++++++++ .../transformer/const_default.dart.expect | 5 -- .../type_flow/transformer/libraries.dart | 22 +++++ .../transformer/libraries.dart.expect | 33 ++++++++ .../transformer/libraries_lib1.lib.dart | 7 ++ .../transformer/libraries_lib2.lib.dart | 7 ++ .../transformer/libraries_lib3.lib.dart | 5 ++ .../lib/create_test.dart.expect | 11 ++- .../lib/decode_test.dart.expect | 11 ++- .../lib/freeze_test.dart.expect | 12 ++- 10 files changed, 186 insertions(+), 8 deletions(-) create mode 100644 pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart create mode 100644 pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart.expect create mode 100644 pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib1.lib.dart create mode 100644 pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib2.lib.dart create mode 100644 pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib3.lib.dart diff --git a/pkg/vm/lib/transformations/type_flow/transformer.dart b/pkg/vm/lib/transformations/type_flow/transformer.dart index 0f2a4d6c61f..3907e4a8127 100644 --- a/pkg/vm/lib/transformations/type_flow/transformer.dart +++ b/pkg/vm/lib/transformations/type_flow/transformer.dart @@ -701,6 +701,7 @@ class AnnotateKernel extends RecursiveVisitor { class TreeShaker { final TypeFlowAnalysis typeFlowAnalysis; final bool treeShakeWriteOnlyFields; + final Set _usedLibraries = new Set(); final Set _usedClasses = new Set(); final Set _classesUsedInType = new Set(); final Set _usedMembers = new Set(); @@ -734,6 +735,7 @@ class TreeShaker { _pass2.transformComponent(component); } + bool isLibraryUsed(Library l) => _usedLibraries.contains(l); bool isClassReferencedFromNativeCode(Class c) => typeFlowAnalysis.nativeCodeOracle.isClassReferencedFromNativeCode(c); bool isClassUsed(Class c) => _usedClasses.contains(c); @@ -772,6 +774,7 @@ class TreeShaker { debugPrint('Class ${c.name} used in type'); } _usedClasses.add(c); + _usedLibraries.add(c.enclosingLibrary); visitIterable(c.supers, typeVisitor); _pass1.transformTypeParameterList(c.typeParameters, c); _pass1.transformExpressionList(c.annotations, c); @@ -794,6 +797,7 @@ class TreeShaker { } _usedClasses.add(enclosingClass); } + _usedLibraries.add(m.enclosingLibrary); FunctionNode? func = null; if (m is Field) { @@ -891,6 +895,7 @@ class TreeShaker { void addUsedTypedef(Typedef typedef) { if (_usedTypedefs.add(typedef)) { + _usedLibraries.add(typedef.enclosingLibrary); typedef.annotations = const []; _pass1.transformTypeParameterList(typedef.typeParameters, typedef); typedef.type?.accept(typeVisitor); @@ -1217,6 +1222,19 @@ class _TreeShakerPass1 extends RemovingTransformer { return node; } + @override + TreeNode visitLoadLibrary(LoadLibrary node, TreeNode? removalSentinel) { + shaker._usedLibraries.add(node.import.targetLibrary); + return node; + } + + @override + TreeNode visitCheckLibraryIsLoaded( + CheckLibraryIsLoaded node, TreeNode? removalSentinel) { + shaker._usedLibraries.add(node.import.targetLibrary); + return node; + } + @override TreeNode visitInstanceInvocation( InstanceInvocation node, TreeNode? removalSentinel) { @@ -1727,8 +1745,44 @@ class _TreeShakerPass2 extends RemovingTransformer { } } + final _libraryExportDeps = >{}; + final _additionalDeps = {}; + + // Returns set of export dependencies of given library. + Set getLibraryExportDeps(Library node) => + _libraryExportDeps[node] ??= calculateLibraryExportDeps(node); + + Set calculateLibraryExportDeps(Library node) { + final processed = {}; + final worklist = []; + final deps = {}; + worklist.add(node); + processed.add(node); + while (worklist.isNotEmpty) { + final lib = worklist.removeLast(); + for (final dep in lib.dependencies) { + if (!dep.isExport) { + continue; + } + final targetLibrary = dep.targetLibrary; + if (processed.add(targetLibrary)) { + if (shaker.isLibraryUsed(targetLibrary)) { + deps.add(targetLibrary); + } else { + worklist.add(targetLibrary); + } + } + } + } + return deps; + } + @override TreeNode visitLibrary(Library node, TreeNode? removalSentinel) { + if (!shaker.isLibraryUsed(node) && node.importUri.scheme != 'dart') { + return removalSentinel!; + } + _additionalDeps.clear(); node.transformOrRemoveChildren(this); // The transformer API does not iterate over `Library.additionalExports`, // so we manually delete the references to shaken nodes. @@ -1746,6 +1800,29 @@ class _TreeShakerPass2 extends RemovingTransformer { return !shaker.isMemberUsed(node as Member); } }); + // Add transitive export dependencies of the removed imported libraries. + // This is needed to maintain connected library graph + // which is critical for calculation of the deferred loading units. + if (_additionalDeps.isNotEmpty) { + for (final dep in node.dependencies) { + _additionalDeps.remove(dep.targetLibrary); + } + for (final lib in _additionalDeps) { + node.addDependency(LibraryDependency.import(lib)); + } + _additionalDeps.clear(); + } + return node; + } + + @override + TreeNode visitLibraryDependency( + LibraryDependency node, TreeNode? removalSentinel) { + final targetLibrary = node.targetLibrary; + if (!shaker.isLibraryUsed(targetLibrary)) { + _additionalDeps.addAll(getLibraryExportDeps(targetLibrary)); + return removalSentinel!; + } return node; } @@ -1991,6 +2068,10 @@ class _TreeShakerConstantVisitor extends ConstantVisitor { @override visitSymbolConstant(SymbolConstant constant) { + final libraryRef = constant.libraryReference; + if (libraryRef != null) { + shaker._usedLibraries.add(libraryRef.asLibrary); + } // The Symbol class and it's _name field are always retained. } diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/const_default.dart.expect b/pkg/vm/testcases/transformations/type_flow/transformer/const_default.dart.expect index d0d1f003303..d202408db9f 100644 --- a/pkg/vm/testcases/transformations/type_flow/transformer/const_default.dart.expect +++ b/pkg/vm/testcases/transformations/type_flow/transformer/const_default.dart.expect @@ -1,11 +1,6 @@ library #lib; import self as self; -import "file:pkg/vm/testcases/transformations/type_flow/transformer/const_default.lib.dart"; - static method main() → dynamic { has-declared-initializer dynamic dyn = null; } -library const_default.lib.dart; -import self as self; - diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart b/pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart new file mode 100644 index 00000000000..aea1faf138b --- /dev/null +++ b/pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart @@ -0,0 +1,22 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'libraries_lib1.lib.dart'; +import 'libraries_lib2.lib.dart'; +import 'libraries_lib3.lib.dart'; + +void unused() { + print(Foo1()); + print(bar1()); +} + +void used() { + print(Foo2()); + print(bar2()); + print(privateSymbol); +} + +main() { + used(); +} diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart.expect b/pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart.expect new file mode 100644 index 00000000000..737764317c4 --- /dev/null +++ b/pkg/vm/testcases/transformations/type_flow/transformer/libraries.dart.expect @@ -0,0 +1,33 @@ +library #lib; +import self as self; +import "dart:core" as core; +import "libraries_lib2.lib.dart" as lib; + +import "file:pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib2.lib.dart"; +import "file:pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib3.lib.dart"; + +static method used() → void { + core::print(new lib::Foo2::•()); + core::print([@vm.inferred-type.metadata=dart.core::_OneByteString (value: "hi")] lib::bar2()); + core::print(#C1); +} +static method main() → dynamic { + self::used(); +} +constants { + #C1 = #file:pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib3.lib.dart::_bazz +} +library libraries_lib2.lib.dart; +import self as self; +import "dart:core" as core; + +class Foo2 extends core::Object { + synthetic constructor •() → self::Foo2 + : super core::Object::•() + ; +} +static method bar2() → core::String + return "hi"; +library libraries_lib3.lib.dart; +import self as self; + diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib1.lib.dart b/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib1.lib.dart new file mode 100644 index 00000000000..6111cb9db6c --- /dev/null +++ b/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib1.lib.dart @@ -0,0 +1,7 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +class Foo1 {} + +String bar1() => 'bye'; diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib2.lib.dart b/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib2.lib.dart new file mode 100644 index 00000000000..f21712508c1 --- /dev/null +++ b/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib2.lib.dart @@ -0,0 +1,7 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +class Foo2 {} + +String bar2() => 'hi'; diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib3.lib.dart b/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib3.lib.dart new file mode 100644 index 00000000000..bf05e09f061 --- /dev/null +++ b/pkg/vm/testcases/transformations/type_flow/transformer/libraries_lib3.lib.dart @@ -0,0 +1,5 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +const privateSymbol = #_bazz; diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/create_test.dart.expect b/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/create_test.dart.expect index 90e1fd07235..b2e29931052 100644 --- a/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/create_test.dart.expect +++ b/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/create_test.dart.expect @@ -5,8 +5,17 @@ import "dart:core" as core; import "package:test_core/scaffolding.dart" as sca; import "package:matcher/src/expect/expect.dart" as exp; -import "package:test/test.dart"; import "file:pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/generated/foo.pb.dart"; +import "package:matcher/src/expect/expect.dart"; +import "package:test_api/hooks.dart"; +import "package:test_core/scaffolding.dart"; +import "package:matcher/src/core_matchers.dart"; +import "package:matcher/src/description.dart"; +import "package:matcher/src/equals_matcher.dart"; +import "package:matcher/src/interfaces.dart"; +import "package:matcher/src/operator_matchers.dart"; +import "package:matcher/src/type_matcher.dart"; +import "package:matcher/src/util.dart"; static method main() → dynamic { pb::FooKeep foo = let final pb::FooKeep #t1 = [@vm.inferred-type.metadata=library file:pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/generated/foo.pb.dart::FooKeep] pb::FooKeep::•() in block { diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/decode_test.dart.expect b/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/decode_test.dart.expect index 8ed9e259fa0..745d8b5db4a 100644 --- a/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/decode_test.dart.expect +++ b/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/decode_test.dart.expect @@ -5,8 +5,17 @@ import "package:test_core/scaffolding.dart" as sca; import "package:matcher/src/expect/expect.dart" as exp; import "dart:core" as core; -import "package:test/test.dart"; import "file:pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/generated/foo.pb.dart"; +import "package:matcher/src/expect/expect.dart"; +import "package:test_api/hooks.dart"; +import "package:test_core/scaffolding.dart"; +import "package:matcher/src/core_matchers.dart"; +import "package:matcher/src/description.dart"; +import "package:matcher/src/equals_matcher.dart"; +import "package:matcher/src/interfaces.dart"; +import "package:matcher/src/operator_matchers.dart"; +import "package:matcher/src/type_matcher.dart"; +import "package:matcher/src/util.dart"; [@vm.inferred-type.metadata=dart.core::_GrowableList]static field core::List buffer = [10, 4, 8, 5, 16, 4, 26, 9, 10, 3, 102, 111, 111, 18, 2, 8, 42, 34, 9, 10, 3, 122, 111, 112, 18, 2, 8, 3, 40, 43, 50, 0, 58, 0]; static method main() → dynamic { diff --git a/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/freeze_test.dart.expect b/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/freeze_test.dart.expect index 6c781c3d04d..619e15030f3 100644 --- a/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/freeze_test.dart.expect +++ b/pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/freeze_test.dart.expect @@ -7,8 +7,18 @@ import "package:protobuf/protobuf.dart" as pro; import "package:matcher/src/expect/expect.dart" as exp; import "package:matcher/src/expect/throws_matcher.dart" as thr; -import "package:test/test.dart"; import "file:pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/generated/foo.pb.dart"; +import "package:matcher/src/expect/expect.dart"; +import "package:matcher/src/expect/throws_matcher.dart"; +import "package:test_api/hooks.dart"; +import "package:test_core/scaffolding.dart"; +import "package:matcher/src/core_matchers.dart"; +import "package:matcher/src/description.dart"; +import "package:matcher/src/equals_matcher.dart"; +import "package:matcher/src/interfaces.dart"; +import "package:matcher/src/operator_matchers.dart"; +import "package:matcher/src/type_matcher.dart"; +import "package:matcher/src/util.dart"; static method main() → dynamic { pb::FooKeep foo = let final pb::FooKeep #t1 = [@vm.inferred-type.metadata=library file:pkg/vm/testcases/transformations/type_flow/transformer/protobuf_handler/lib/generated/foo.pb.dart::FooKeep] pb::FooKeep::•() in block {