From c2500c3aa0fed8fdc2aa35faecba5aa2ea33ba9a Mon Sep 17 00:00:00 2001 From: "nweiz@google.com" Date: Tue, 4 Jun 2013 21:05:00 +0000 Subject: [PATCH] Move pub over to using the pub.dartlang.org API v2. R=rnystrom@google.com Review URL: https://codereview.chromium.org//16351003 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@23615 260f80e4-7a28-3924-810f-c04153c831b5 --- .../_internal/pub/lib/src/command_lish.dart | 6 +- .../pub/lib/src/command_uploader.dart | 15 +- .../_internal/pub/lib/src/hosted_source.dart | 14 +- sdk/lib/_internal/pub/lib/src/http.dart | 17 +- sdk/lib/_internal/pub/lib/src/pubspec.dart | 250 +++++++++--------- ...pies_dart_js_next_to_entrypoints_test.dart | 17 +- .../test/hosted/version_negotiation_test.dart | 68 +++++ .../upload_form_provides_an_error_test.dart | 2 +- ...pload_form_provides_invalid_json_test.dart | 2 +- sdk/lib/_internal/pub/test/lish/utils.dart | 2 +- ..._credentials_authenticates_again_test.dart | 2 +- ...redentials_does_not_authenticate_test.dart | 2 +- ...efresh_token_authenticates_again_test.dart | 2 +- ..._credentials_refreshes_and_saves_test.dart | 2 +- ...efresh_token_authenticates_again_test.dart | 2 +- ...henticates_and_saves_credentials_test.dart | 2 +- ..._credentials_authenticates_again_test.dart | 2 +- .../_internal/pub/test/pub_uploader_test.dart | 14 +- sdk/lib/_internal/pub/test/test_pub.dart | 104 ++++++-- .../pub/test/validator/dependency_test.dart | 5 +- 20 files changed, 347 insertions(+), 183 deletions(-) create mode 100644 sdk/lib/_internal/pub/test/hosted/version_negotiation_test.dart diff --git a/sdk/lib/_internal/pub/lib/src/command_lish.dart b/sdk/lib/_internal/pub/lib/src/command_lish.dart index 20bbdd8e528..8e4e09cf6aa 100644 --- a/sdk/lib/_internal/pub/lib/src/command_lish.dart +++ b/sdk/lib/_internal/pub/lib/src/command_lish.dart @@ -54,8 +54,8 @@ class LishCommand extends PubCommand { return log.progress('Uploading', () { // TODO(nweiz): Cloud Storage can provide an XML-formatted error. We // should report that error and exit. - var newUri = server.resolve("/packages/versions/new.json"); - return client.get(newUri).then((response) { + var newUri = server.resolve("/api/packages/versions/new"); + return client.get(newUri, headers: PUB_API_HEADERS).then((response) { var parameters = parseJsonResponse(response); var url = _expectField(parameters, 'url', response); @@ -78,7 +78,7 @@ class LishCommand extends PubCommand { var location = response.headers['location']; if (location == null) throw new PubHttpException(response); return location; - }).then((location) => client.get(location)) + }).then((location) => client.get(location, headers: PUB_API_HEADERS)) .then(handleJsonSuccess); }); }).catchError((error) { diff --git a/sdk/lib/_internal/pub/lib/src/command_uploader.dart b/sdk/lib/_internal/pub/lib/src/command_uploader.dart index c3b9be8012c..cb6bf8b1e81 100644 --- a/sdk/lib/_internal/pub/lib/src/command_uploader.dart +++ b/sdk/lib/_internal/pub/lib/src/command_uploader.dart @@ -63,13 +63,16 @@ class UploaderCommand extends PubCommand { var uploader = commandOptions.rest[0]; return oauth2.withClient(cache, (client) { if (command == 'add') { - var url = server.resolve("/packages/${Uri.encodeComponent(package)}" - "/uploaders.json"); - return client.post(url, fields: {"email": uploader}); + var url = server.resolve("/api/packages/" + "${Uri.encodeComponent(package)}/uploaders"); + return client.post(url, + headers: PUB_API_HEADERS, + fields: {"email": uploader}); } else { // command == 'remove' - var url = server.resolve("/packages/${Uri.encodeComponent(package)}" - "/uploaders/${Uri.encodeComponent(uploader)}.json"); - return client.delete(url); + var url = server.resolve("/api/packages/" + "${Uri.encodeComponent(package)}/uploaders/" + "${Uri.encodeComponent(uploader)}"); + return client.delete(url, headers: PUB_API_HEADERS); } }); }).then(handleJsonSuccess) diff --git a/sdk/lib/_internal/pub/lib/src/hosted_source.dart b/sdk/lib/_internal/pub/lib/src/hosted_source.dart index a5c99924d6e..beabfd1d3d6 100644 --- a/sdk/lib/_internal/pub/lib/src/hosted_source.dart +++ b/sdk/lib/_internal/pub/lib/src/hosted_source.dart @@ -34,13 +34,13 @@ class HostedSource extends Source { /// site. Future> getVersions(String name, description) { var url = _makeUrl(description, - (server, package) => "$server/packages/$package.json"); + (server, package) => "$server/api/packages/$package"); log.io("Get versions from $url."); - return httpClient.read(url).then((body) { + return httpClient.read(url, headers: PUB_API_HEADERS).then((body) { var doc = json.parse(body); return doc['versions'] - .map((version) => new Version.parse(version)) + .map((version) => new Version.parse(version['version'])) .toList(); }).catchError((ex) { var parsed = _parseDescription(description); @@ -53,15 +53,17 @@ class HostedSource extends Source { Future describeUncached(PackageId id) { // Request it from the server. var url = _makeVersionUrl(id, (server, package, version) => - "$server/packages/$package/versions/$version.yaml"); + "$server/api/packages/$package/versions/$version"); log.io("Describe package at $url."); - return httpClient.read(url).then((yaml) { + return httpClient.read(url, headers: PUB_API_HEADERS).then((version) { + version = json.parse(version); + // TODO(rnystrom): After this is pulled down, we could place it in // a secondary cache of just pubspecs. This would let us have a // persistent cache for pubspecs for packages that haven't actually // been installed. - return new Pubspec.parse(null, yaml, systemCache.sources); + return new Pubspec.fromMap(version['pubspec'], systemCache.sources); }).catchError((ex) { var parsed = _parseDescription(id.description); _throwFriendlyError(ex, id, parsed.last); diff --git a/sdk/lib/_internal/pub/lib/src/http.dart b/sdk/lib/_internal/pub/lib/src/http.dart index 00fa9522faa..e7da34c6c61 100644 --- a/sdk/lib/_internal/pub/lib/src/http.dart +++ b/sdk/lib/_internal/pub/lib/src/http.dart @@ -14,6 +14,7 @@ import 'package:http/http.dart' as http; import 'io.dart'; import 'log.dart' as log; import 'oauth2.dart' as oauth2; +import 'sdk.dart' as sdk; import 'utils.dart'; // TODO(nweiz): make this configurable @@ -24,6 +25,13 @@ final HTTP_TIMEOUT = 30 * 1000; /// Headers and field names that should be censored in the log output. final _CENSORED_FIELDS = const ['refresh_token', 'authorization']; +/// Headers required for pub.dartlang.org API requests. +/// +/// The Accept header tells pub.dartlang.org which version of the API we're +/// expecting, so it can either serve that version or give us a 406 error if +/// it's not supported. +final PUB_API_HEADERS = const {'Accept': 'application/vnd.pub.v2+json'}; + /// Whether dart:io's SecureSocket has been initialized with pub's resources /// yet. bool _initializedSecureSocket = false; @@ -54,8 +62,6 @@ class PubHttpClient extends http.BaseClient { stackTrace = localStackTrace; } - // TODO(nweiz): Ideally the timeout would extend to reading from the - // response input stream, but until issue 3657 is fixed that's not feasible. return timeout(inner.send(request).then((streamedResponse) { _logResponse(streamedResponse); @@ -69,6 +75,13 @@ class PubHttpClient extends http.BaseClient { return streamedResponse; } + if (status == 406 && + request.headers['Accept'] == PUB_API_HEADERS['Accept']) { + fail("Pub ${sdk.version} is incompatible with the current version of " + "${request.url.host}.\n" + "Upgrade pub to the latest version and try again."); + } + return http.Response.fromStream(streamedResponse).then((response) { throw new PubHttpException(response); }); diff --git a/sdk/lib/_internal/pub/lib/src/pubspec.dart b/sdk/lib/_internal/pub/lib/src/pubspec.dart index e01cab15831..2100cc60171 100644 --- a/sdk/lib/_internal/pub/lib/src/pubspec.dart +++ b/sdk/lib/_internal/pub/lib/src/pubspec.dart @@ -74,6 +74,13 @@ class Pubspec { bool get isEmpty => name == null && version == Version.none && dependencies.isEmpty; + /// Returns a Pubspec object for an already-parsed map representing its + /// contents. + /// + /// This will validate that [contents] is a valid pubspec. + factory Pubspec.fromMap(Map contents, SourceRegistry sources) => + _parseMap(null, contents, sources); + // TODO(rnystrom): Instead of allowing a null argument here, split this up // into load(), parse(), and _parse() like LockFile does. /// Parses the pubspec stored at [filePath] whose text is [contents]. If the @@ -82,9 +89,6 @@ class Pubspec { /// file system. factory Pubspec.parse(String filePath, String contents, SourceRegistry sources) { - var name = null; - var version = Version.none; - if (contents.trim() == '') return new Pubspec.empty(); var parsedPubspec = loadYaml(contents); @@ -94,123 +98,7 @@ class Pubspec { throw new FormatException('The pubspec must be a YAML mapping.'); } - if (parsedPubspec.containsKey('name')) { - name = parsedPubspec['name']; - if (name is! String) { - throw new FormatException( - 'The pubspec "name" field should be a string, but was "$name".'); - } - } - - if (parsedPubspec.containsKey('version')) { - version = new Version.parse(parsedPubspec['version']); - } - - var dependencies = _parseDependencies(filePath, sources, - parsedPubspec['dependencies']); - - var devDependencies = _parseDependencies(filePath, sources, - parsedPubspec['dev_dependencies']); - - // Make sure the same package doesn't appear as both a regular and dev - // dependency. - var dependencyNames = dependencies.map((dep) => dep.name).toSet(); - var collisions = dependencyNames.intersection( - devDependencies.map((dep) => dep.name).toSet()); - - if (!collisions.isEmpty) { - var packageNames; - if (collisions.length == 1) { - packageNames = 'Package "${collisions.first}"'; - } else { - var names = collisions.toList(); - names.sort(); - var buffer = new StringBuffer(); - buffer.write("Packages "); - for (var i = 0; i < names.length; i++) { - buffer.write('"'); - buffer.write(names[i]); - buffer.write('"'); - if (i == names.length - 2) { - buffer.write(", "); - } else if (i == names.length - 1) { - buffer.write(", and "); - } - } - - packageNames = buffer.toString(); - } - throw new FormatException( - '$packageNames cannot appear in both "dependencies" and ' - '"dev_dependencies".'); - } - - var environmentYaml = parsedPubspec['environment']; - var sdkConstraint = VersionConstraint.any; - if (environmentYaml != null) { - if (environmentYaml is! Map) { - throw new FormatException( - 'The pubspec "environment" field should be a map, but was ' - '"$environmentYaml".'); - } - - var sdkYaml = environmentYaml['sdk']; - if (sdkYaml is! String) { - throw new FormatException( - 'The "sdk" field of "environment" should be a string, but was ' - '"$sdkYaml".'); - } - - sdkConstraint = new VersionConstraint.parse(sdkYaml); - } - var environment = new PubspecEnvironment(sdkConstraint); - - // Even though the pub app itself doesn't use these fields, we validate - // them here so that users find errors early before they try to upload to - // the server: - // TODO(rnystrom): We should split this validation into separate layers: - // 1. Stuff that is required in any pubspec to perform any command. Things - // like "must have a name". That should go here. - // 2. Stuff that is required to upload a package. Things like "homepage - // must use a valid scheme". That should go elsewhere. pub upload should - // call it, and we should provide a separate command to show the user, - // and also expose it to the editor in some way. - - if (parsedPubspec.containsKey('homepage')) { - _validateFieldUrl(parsedPubspec['homepage'], 'homepage'); - } - if (parsedPubspec.containsKey('documentation')) { - _validateFieldUrl(parsedPubspec['documentation'], 'documentation'); - } - - if (parsedPubspec.containsKey('author') && - parsedPubspec['author'] is! String) { - throw new FormatException( - 'The "author" field should be a string, but was ' - '${parsedPubspec["author"]}.'); - } - - if (parsedPubspec.containsKey('authors')) { - var authors = parsedPubspec['authors']; - if (authors is List) { - // All of the elements must be strings. - if (!authors.every((author) => author is String)) { - throw new FormatException('The "authors" field should be a string ' - 'or a list of strings, but was "$authors".'); - } - } else if (authors is! String) { - throw new FormatException('The pubspec "authors" field should be a ' - 'string or a list of strings, but was "$authors".'); - } - - if (parsedPubspec.containsKey('author')) { - throw new FormatException('A pubspec should not have both an "author" ' - 'and an "authors" field.'); - } - } - - return new Pubspec(name, version, dependencies, devDependencies, - environment, parsedPubspec); + return _parseMap(filePath, parsedPubspec, sources); } } @@ -231,6 +119,128 @@ void _validateFieldUrl(url, String field) { } } +Pubspec _parseMap(String filePath, Map map, SourceRegistry sources) { + var name = null; + var version = Version.none; + + if (map.containsKey('name')) { + name = map['name']; + if (name is! String) { + throw new FormatException( + 'The pubspec "name" field should be a string, but was "$name".'); + } + } + + if (map.containsKey('version')) { + version = new Version.parse(map['version']); + } + + var dependencies = _parseDependencies(filePath, sources, + map['dependencies']); + + var devDependencies = _parseDependencies(filePath, sources, + map['dev_dependencies']); + + // Make sure the same package doesn't appear as both a regular and dev + // dependency. + var dependencyNames = dependencies.map((dep) => dep.name).toSet(); + var collisions = dependencyNames.intersection( + devDependencies.map((dep) => dep.name).toSet()); + + if (!collisions.isEmpty) { + var packageNames; + if (collisions.length == 1) { + packageNames = 'Package "${collisions.first}"'; + } else { + var names = collisions.toList(); + names.sort(); + var buffer = new StringBuffer(); + buffer.write("Packages "); + for (var i = 0; i < names.length; i++) { + buffer.write('"'); + buffer.write(names[i]); + buffer.write('"'); + if (i == names.length - 2) { + buffer.write(", "); + } else if (i == names.length - 1) { + buffer.write(", and "); + } + } + + packageNames = buffer.toString(); + } + throw new FormatException( + '$packageNames cannot appear in both "dependencies" and ' + '"dev_dependencies".'); + } + + var environmentYaml = map['environment']; + var sdkConstraint = VersionConstraint.any; + if (environmentYaml != null) { + if (environmentYaml is! Map) { + throw new FormatException( + 'The pubspec "environment" field should be a map, but was ' + '"$environmentYaml".'); + } + + var sdkYaml = environmentYaml['sdk']; + if (sdkYaml is! String) { + throw new FormatException( + 'The "sdk" field of "environment" should be a string, but was ' + '"$sdkYaml".'); + } + + sdkConstraint = new VersionConstraint.parse(sdkYaml); + } + var environment = new PubspecEnvironment(sdkConstraint); + + // Even though the pub app itself doesn't use these fields, we validate + // them here so that users find errors early before they try to upload to + // the server: + // TODO(rnystrom): We should split this validation into separate layers: + // 1. Stuff that is required in any pubspec to perform any command. Things + // like "must have a name". That should go here. + // 2. Stuff that is required to upload a package. Things like "homepage + // must use a valid scheme". That should go elsewhere. pub upload should + // call it, and we should provide a separate command to show the user, + // and also expose it to the editor in some way. + + if (map.containsKey('homepage')) { + _validateFieldUrl(map['homepage'], 'homepage'); + } + if (map.containsKey('documentation')) { + _validateFieldUrl(map['documentation'], 'documentation'); + } + + if (map.containsKey('author') && map['author'] is! String) { + throw new FormatException( + 'The "author" field should be a string, but was ' + '${map["author"]}.'); + } + + if (map.containsKey('authors')) { + var authors = map['authors']; + if (authors is List) { + // All of the elements must be strings. + if (!authors.every((author) => author is String)) { + throw new FormatException('The "authors" field should be a string ' + 'or a list of strings, but was "$authors".'); + } + } else if (authors is! String) { + throw new FormatException('The pubspec "authors" field should be a ' + 'string or a list of strings, but was "$authors".'); + } + + if (map.containsKey('author')) { + throw new FormatException('A pubspec should not have both an "author" ' + 'and an "authors" field.'); + } + } + + return new Pubspec(name, version, dependencies, devDependencies, + environment, map); +} + List _parseDependencies(String pubspecPath, SourceRegistry sources, yaml) { var dependencies = []; diff --git a/sdk/lib/_internal/pub/test/deploy/copies_dart_js_next_to_entrypoints_test.dart b/sdk/lib/_internal/pub/test/deploy/copies_dart_js_next_to_entrypoints_test.dart index 7eecff3f9cd..417233b4321 100644 --- a/sdk/lib/_internal/pub/test/deploy/copies_dart_js_next_to_entrypoints_test.dart +++ b/sdk/lib/_internal/pub/test/deploy/copies_dart_js_next_to_entrypoints_test.dart @@ -19,11 +19,24 @@ main() { currentSchedule.timeout *= 3; serve([ + d.dir('api', [ + d.dir('packages', [ + d.file('browser', json.stringify({ + 'versions': [packageVersionApiMap(packageMap('browser', '1.0.0'))] + })), + d.dir('browser', [ + d.dir('versions', [ + d.file('1.0.0', json.stringify( + packageVersionApiMap( + packageMap('browser', '1.0.0'), + full: true))) + ]) + ]) + ]) + ]), d.dir('packages', [ - d.file('browser.json', json.stringify({'versions': ['1.0.0']})), d.dir('browser', [ d.dir('versions', [ - d.file('1.0.0.yaml', yaml(packageMap("browser", "1.0.0"))), d.tar('1.0.0.tar.gz', [ d.file('pubspec.yaml', yaml(packageMap("browser", "1.0.0"))), d.dir('lib', [ diff --git a/sdk/lib/_internal/pub/test/hosted/version_negotiation_test.dart b/sdk/lib/_internal/pub/test/hosted/version_negotiation_test.dart new file mode 100644 index 00000000000..2e51f7b2b4c --- /dev/null +++ b/sdk/lib/_internal/pub/test/hosted/version_negotiation_test.dart @@ -0,0 +1,68 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library pub_tests; + +import 'dart:io'; + +import 'package:scheduled_test/scheduled_server.dart'; +import 'package:scheduled_test/scheduled_test.dart'; + +import '../descriptor.dart' as d; +import '../test_pub.dart'; + +main() { + initConfig(); + + forBothPubInstallAndUpdate((command) { + integration('sends the correct Accept header', () { + var server = new ScheduledServer(); + + d.appDir([{ + "hosted": { + "name": "foo", + "url": server.url.then((url) => url.toString()) + } + }]).create(); + + var pub = startPub(args: [command.name]); + + server.handle('GET', '/api/packages/foo', (request) { + expect(request.headers['Accept'], ['application/vnd.pub.v2+json']); + }); + + pub.kill(); + }); + + integration('prints a friendly error if the version is out-of-date', () { + var server = new ScheduledServer(); + + d.appDir([{ + "hosted": { + "name": "foo", + "url": server.url.then((url) => url.toString()) + } + }]).create(); + + var pub = startPub(args: [command.name]); + + server.handle('GET', '/api/packages/foo', (request) { + request.response.statusCode = 406; + request.response.close(); + }); + + // TODO(nweiz): this shouldn't request the versions twice (issue 11077). + server.handle('GET', '/api/packages/foo', (request) { + request.response.statusCode = 406; + request.response.close(); + }); + + pub.shouldExit(1); + + expect(pub.remainingStderr(), completion(equals( + "Pub 0.1.2+3 is incompatible with the current version of localhost.\n" + "Upgrade pub to the latest version and try again."))); + }); + }); +} diff --git a/sdk/lib/_internal/pub/test/lish/upload_form_provides_an_error_test.dart b/sdk/lib/_internal/pub/test/lish/upload_form_provides_an_error_test.dart index 6a3b2a10606..c2d753b1783 100644 --- a/sdk/lib/_internal/pub/test/lish/upload_form_provides_an_error_test.dart +++ b/sdk/lib/_internal/pub/test/lish/upload_form_provides_an_error_test.dart @@ -22,7 +22,7 @@ main() { confirmPublish(pub); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { request.response.statusCode = 400; request.response.write(json.stringify({ 'error': {'message': 'your request sucked'} diff --git a/sdk/lib/_internal/pub/test/lish/upload_form_provides_invalid_json_test.dart b/sdk/lib/_internal/pub/test/lish/upload_form_provides_invalid_json_test.dart index 3f2c4866871..3a1b2cbc695 100644 --- a/sdk/lib/_internal/pub/test/lish/upload_form_provides_invalid_json_test.dart +++ b/sdk/lib/_internal/pub/test/lish/upload_form_provides_invalid_json_test.dart @@ -22,7 +22,7 @@ main() { confirmPublish(pub); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { request.response.write('{not json'); request.response.close(); }); diff --git a/sdk/lib/_internal/pub/test/lish/utils.dart b/sdk/lib/_internal/pub/test/lish/utils.dart index ba921e2b07f..6ee3f5e608c 100644 --- a/sdk/lib/_internal/pub/test/lish/utils.dart +++ b/sdk/lib/_internal/pub/test/lish/utils.dart @@ -14,7 +14,7 @@ import '../../lib/src/io.dart'; import '../test_pub.dart'; void handleUploadForm(ScheduledServer server, [Map body]) { - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { return server.url.then((url) { expect(request.headers.value('authorization'), equals('Bearer access token')); diff --git a/sdk/lib/_internal/pub/test/oauth2/with_a_malformed_credentials_authenticates_again_test.dart b/sdk/lib/_internal/pub/test/oauth2/with_a_malformed_credentials_authenticates_again_test.dart index e207d1b9ef6..574117baebf 100644 --- a/sdk/lib/_internal/pub/test/oauth2/with_a_malformed_credentials_authenticates_again_test.dart +++ b/sdk/lib/_internal/pub/test/oauth2/with_a_malformed_credentials_authenticates_again_test.dart @@ -29,7 +29,7 @@ main() { confirmPublish(pub); authorizePub(pub, server, "new access token"); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { expect(request.headers.value('authorization'), equals('Bearer new access token')); diff --git a/sdk/lib/_internal/pub/test/oauth2/with_a_pre_existing_credentials_does_not_authenticate_test.dart b/sdk/lib/_internal/pub/test/oauth2/with_a_pre_existing_credentials_does_not_authenticate_test.dart index 853d7d9bcd6..35653ffc094 100644 --- a/sdk/lib/_internal/pub/test/oauth2/with_a_pre_existing_credentials_does_not_authenticate_test.dart +++ b/sdk/lib/_internal/pub/test/oauth2/with_a_pre_existing_credentials_does_not_authenticate_test.dart @@ -24,7 +24,7 @@ main() { var pub = startPublish(server); confirmPublish(pub); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { expect(request.headers.value('authorization'), equals('Bearer access token')); diff --git a/sdk/lib/_internal/pub/test/oauth2/with_a_server_rejected_refresh_token_authenticates_again_test.dart b/sdk/lib/_internal/pub/test/oauth2/with_a_server_rejected_refresh_token_authenticates_again_test.dart index 1a103e98bcc..bbfdc6472f5 100644 --- a/sdk/lib/_internal/pub/test/oauth2/with_a_server_rejected_refresh_token_authenticates_again_test.dart +++ b/sdk/lib/_internal/pub/test/oauth2/with_a_server_rejected_refresh_token_authenticates_again_test.dart @@ -45,7 +45,7 @@ main() { expect(pub.nextLine(), completion(matches(r'Uploading\.\.\.+'))); authorizePub(pub, server, 'new access token'); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { expect(request.headers.value('authorization'), equals('Bearer new access token')); diff --git a/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_refreshes_and_saves_test.dart b/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_refreshes_and_saves_test.dart index 7622e474863..17eed6f9730 100644 --- a/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_refreshes_and_saves_test.dart +++ b/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_refreshes_and_saves_test.dart @@ -45,7 +45,7 @@ main() { }); }); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { expect(request.headers.value('authorization'), equals('Bearer new access token')); diff --git a/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_without_a_refresh_token_authenticates_again_test.dart b/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_without_a_refresh_token_authenticates_again_test.dart index a844eff31b6..0961868c659 100644 --- a/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_without_a_refresh_token_authenticates_again_test.dart +++ b/sdk/lib/_internal/pub/test/oauth2/with_an_expired_credentials_without_a_refresh_token_authenticates_again_test.dart @@ -32,7 +32,7 @@ main() { "packages has expired and can't be automatically refreshed."))); authorizePub(pub, server, "new access token"); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { expect(request.headers.value('authorization'), equals('Bearer new access token')); diff --git a/sdk/lib/_internal/pub/test/oauth2/with_no_credentials_authenticates_and_saves_credentials_test.dart b/sdk/lib/_internal/pub/test/oauth2/with_no_credentials_authenticates_and_saves_credentials_test.dart index ebba5dc1c0a..1790e859e6c 100644 --- a/sdk/lib/_internal/pub/test/oauth2/with_no_credentials_authenticates_and_saves_credentials_test.dart +++ b/sdk/lib/_internal/pub/test/oauth2/with_no_credentials_authenticates_and_saves_credentials_test.dart @@ -24,7 +24,7 @@ main() { confirmPublish(pub); authorizePub(pub, server); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { expect(request.headers.value('authorization'), equals('Bearer access token')); diff --git a/sdk/lib/_internal/pub/test/oauth2/with_server_rejected_credentials_authenticates_again_test.dart b/sdk/lib/_internal/pub/test/oauth2/with_server_rejected_credentials_authenticates_again_test.dart index a5a8e046b5f..6457091fcb3 100644 --- a/sdk/lib/_internal/pub/test/oauth2/with_server_rejected_credentials_authenticates_again_test.dart +++ b/sdk/lib/_internal/pub/test/oauth2/with_server_rejected_credentials_authenticates_again_test.dart @@ -25,7 +25,7 @@ main() { confirmPublish(pub); - server.handle('GET', '/packages/versions/new.json', (request) { + server.handle('GET', '/api/packages/versions/new', (request) { var response = request.response; response.statusCode = 401; response.headers.set('www-authenticate', 'Bearer error="invalid_token",' diff --git a/sdk/lib/_internal/pub/test/pub_uploader_test.dart b/sdk/lib/_internal/pub/test/pub_uploader_test.dart index 7da77c2703f..741abf0ca70 100644 --- a/sdk/lib/_internal/pub/test/pub_uploader_test.dart +++ b/sdk/lib/_internal/pub/test/pub_uploader_test.dart @@ -58,7 +58,7 @@ main() { d.credentialsFile(server, 'access token').create(); var pub = startPubUploader(server, ['--package', 'pkg', 'add', 'email']); - server.handle('POST', '/packages/pkg/uploaders.json', (request) { + server.handle('POST', '/api/packages/pkg/uploaders', (request) { expect(new ByteStream(request).toBytes().then((bodyBytes) { expect(new String.fromCharCodes(bodyBytes), equals('email=email')); @@ -80,7 +80,7 @@ main() { d.credentialsFile(server, 'access token').create(); var pub = startPubUploader(server, ['--package', 'pkg', 'remove', 'email']); - server.handle('DELETE', '/packages/pkg/uploaders/email.json', (request) { + server.handle('DELETE', '/api/packages/pkg/uploaders/email', (request) { request.response.headers.contentType = new ContentType("application", "json"); request.response.write(json.stringify({ @@ -100,7 +100,7 @@ main() { d.credentialsFile(server, 'access token').create(); var pub = startPubUploader(server, ['add', 'email']); - server.handle('POST', '/packages/test_pkg/uploaders.json', (request) { + server.handle('POST', '/api/packages/test_pkg/uploaders', (request) { request.response.headers.contentType = new ContentType("application", "json"); request.response.write(json.stringify({ @@ -118,7 +118,7 @@ main() { d.credentialsFile(server, 'access token').create(); var pub = startPubUploader(server, ['--package', 'pkg', 'add', 'email']); - server.handle('POST', '/packages/pkg/uploaders.json', (request) { + server.handle('POST', '/api/packages/pkg/uploaders', (request) { request.response.statusCode = 400; request.response.headers.contentType = new ContentType("application", "json"); @@ -138,7 +138,7 @@ main() { var pub = startPubUploader(server, ['--package', 'pkg', 'remove', 'e/mail']); - server.handle('DELETE', '/packages/pkg/uploaders/e%2Fmail.json', (request) { + server.handle('DELETE', '/api/packages/pkg/uploaders/e%2Fmail', (request) { request.response.statusCode = 400; request.response.headers.contentType = new ContentType("application", "json"); @@ -157,7 +157,7 @@ main() { d.credentialsFile(server, 'access token').create(); var pub = startPubUploader(server, ['--package', 'pkg', 'add', 'email']); - server.handle('POST', '/packages/pkg/uploaders.json', (request) { + server.handle('POST', '/api/packages/pkg/uploaders', (request) { request.response.write("{not json"); request.response.close(); }); @@ -172,7 +172,7 @@ main() { d.credentialsFile(server, 'access token').create(); var pub = startPubUploader(server, ['--package', 'pkg', 'remove', 'email']); - server.handle('DELETE', '/packages/pkg/uploaders/email.json', (request) { + server.handle('DELETE', '/api/packages/pkg/uploaders/email', (request) { request.response.write("{not json"); request.response.close(); }); diff --git a/sdk/lib/_internal/pub/test/test_pub.dart b/sdk/lib/_internal/pub/test/test_pub.dart index 1a784e73c31..0414397d39f 100644 --- a/sdk/lib/_internal/pub/test/test_pub.dart +++ b/sdk/lib/_internal/pub/test/test_pub.dart @@ -147,16 +147,26 @@ Future _closeServer() { return sleep(10); } -/// The [d.DirectoryDescriptor] describing the server layout of packages that -/// are being served via [servePackages]. This is `null` if [servePackages] has -/// not yet been called for this test. +/// The [d.DirectoryDescriptor] describing the server layout of `/api/packages` +/// on the test server. +/// +/// This contains metadata for packages that are being served via +/// [servePackages]. It's `null` if [servePackages] has not yet been called for +/// this test. +d.DirectoryDescriptor _servedApiPackageDir; + +/// The [d.DirectoryDescriptor] describing the server layout of `/packages` on +/// the test server. +/// +/// This contains the tarballs for packages that are being served via +/// [servePackages]. It's `null` if [servePackages] has not yet been called for +/// this test. d.DirectoryDescriptor _servedPackageDir; -/// A map from package names to version numbers to YAML-serialized pubspecs for -/// those packages. This represents the packages currently being served by -/// [servePackages], and is `null` if [servePackages] has not yet been called -/// for this test. -Map> _servedPackages; +/// A map from package names to parsed pubspec maps for those packages. This +/// represents the packages currently being served by [servePackages], and is +/// `null` if [servePackages] has not yet been called for this test. +Map> _servedPackages; /// Creates an HTTP server that replicates the structure of pub.dartlang.org. /// [pubspecs] is a list of unserialized pubspecs representing the packages to @@ -166,12 +176,17 @@ Map> _servedPackages; /// are being served. Previous packages will continue to be served. void servePackages(List pubspecs) { if (_servedPackages == null || _servedPackageDir == null) { - _servedPackages = >{}; + _servedPackages = >{}; + _servedApiPackageDir = d.dir('packages', []); _servedPackageDir = d.dir('packages', []); - serve([_servedPackageDir]); + serve([ + d.dir('api', [_servedApiPackageDir]), + _servedPackageDir + ]); currentSchedule.onComplete.schedule(() { _servedPackages = null; + _servedApiPackageDir = null; _servedPackageDir = null; }, 'cleaning up served packages'); } @@ -181,28 +196,36 @@ void servePackages(List pubspecs) { for (var spec in resolvedPubspecs) { var name = spec['name']; var version = spec['version']; - var versions = _servedPackages.putIfAbsent( - name, () => {}); - versions[version] = yaml(spec); + var versions = _servedPackages.putIfAbsent(name, () => []); + versions.add(spec); } + _servedApiPackageDir.contents.clear(); _servedPackageDir.contents.clear(); for (var name in _servedPackages.keys) { - var versions = _servedPackages[name].keys.toList(); - _servedPackageDir.contents.addAll([ - d.file('$name.json', json.stringify({'versions': versions})), + _servedApiPackageDir.contents.addAll([ + d.file('$name', json.stringify({ + 'name': name, + 'uploaders': ['nweiz@google.com'], + 'versions': _servedPackages[name].map(packageVersionApiMap).toList() + })), d.dir(name, [ - d.dir('versions', flatten(versions.map((version) { - return [ - d.file('$version.yaml', _servedPackages[name][version]), - d.tar('$version.tar.gz', [ - d.file('pubspec.yaml', _servedPackages[name][version]), - d.libDir(name, '$name $version') - ]) - ]; - }))) + d.dir('versions', _servedPackages[name].map((pubspec) { + return d.file(pubspec['version'], json.stringify( + packageVersionApiMap(pubspec, full: true))); + })) ]) ]); + + _servedPackageDir.contents.add(d.dir(name, [ + d.dir('versions', _servedPackages[name].map((pubspec) { + var version = pubspec['version']; + return d.tar('$version.tar.gz', [ + d.file('pubspec.yaml', json.stringify(pubspec)), + d.libDir(name, '$name $version') + ]); + })) + ])); } }); }, 'initializing the package server'); @@ -632,6 +655,37 @@ String _packageName(String sourceName, description) { } } +/// Returns a Map in the format used by the pub.dartlang.org API to represent a +/// package version. +/// +/// [pubspec] is the parsed pubspec of the package version. If [full] is true, +/// this returns the complete map, including metadata that's only included when +/// requesting the package version directly. +Map packageVersionApiMap(Map pubspec, {bool full: false}) { + var name = pubspec['name']; + var version = pubspec['version']; + var map = { + 'pubspec': pubspec, + 'version': version, + 'url': '/api/packages/$name/versions/$version', + 'archive_url': '/packages/$name/versions/$version.tar.gz', + 'new_dartdoc_url': '/api/packages/$name/versions/$version' + '/new_dartdoc', + 'package_url': '/api/packages/$name' + }; + + if (full) { + mapAddAll(map, { + 'downloads': 0, + 'created': '2012-09-25T18:38:28.685260', + 'libraries': ['$name.dart'], + 'uploader': ['nweiz@google.com'] + }); + } + + return map; +} + /// Compares the [actual] output from running pub with [expected]. For [String] /// patterns, ignores leading and trailing whitespace differences and tries to /// report the offending difference in a nice way. For other [Pattern]s, just diff --git a/sdk/lib/_internal/pub/test/validator/dependency_test.dart b/sdk/lib/_internal/pub/test/validator/dependency_test.dart index 9eba069edb8..89f1bf78bad 100644 --- a/sdk/lib/_internal/pub/test/validator/dependency_test.dart +++ b/sdk/lib/_internal/pub/test/validator/dependency_test.dart @@ -35,7 +35,7 @@ expectDependencyValidationWarning(String warning) { setUpDependency(Map dep, {List hostedVersions}) { useMockClient(new MockClient((request) { expect(request.method, equals("GET")); - expect(request.url.path, equals("/packages/foo.json")); + expect(request.url.path, equals("/api/packages/foo")); if (hostedVersions == null) { return new Future.value(new http.Response("not found", 404)); @@ -43,7 +43,8 @@ setUpDependency(Map dep, {List hostedVersions}) { return new Future.value(new http.Response(json.stringify({ "name": "foo", "uploaders": ["nweiz@google.com"], - "versions": hostedVersions + "versions": hostedVersions.map((version) => + packageVersionApiMap(packageMap('foo', version))).toList() }), 200)); } }));