diff --git a/pkg/analyzer_experimental/bin/analyzer.dart b/pkg/analyzer_experimental/bin/analyzer.dart index 119376fb0678..d3c59f38cb7f 100644 --- a/pkg/analyzer_experimental/bin/analyzer.dart +++ b/pkg/analyzer_experimental/bin/analyzer.dart @@ -80,7 +80,7 @@ class BatchRunner { ErrorSeverity batchResult = ErrorSeverity.NONE; // read line from stdin Stream cmdLine = stdin - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); var subscription = cmdLine.listen((String line) { // may be finish diff --git a/pkg/analyzer_experimental/bin/formatter.dart b/pkg/analyzer_experimental/bin/formatter.dart index dd251803dec7..56734261e6c7 100755 --- a/pkg/analyzer_experimental/bin/formatter.dart +++ b/pkg/analyzer_experimental/bin/formatter.dart @@ -37,7 +37,7 @@ _formatFile(path) { var buffer = new StringBuffer(); var file = new File(path); file.openRead() - .transform(new StringDecoder()) + .transform(UTF8.decoder) .listen((data) => buffer.write(data), onError: (error) => print('Error, could not open "$path"'), onDone: () => print(_formatCU(buffer.toString()))); diff --git a/pkg/barback/lib/src/asset.dart b/pkg/barback/lib/src/asset.dart index 05a2202dfa61..db7a5ea2ca8c 100644 --- a/pkg/barback/lib/src/asset.dart +++ b/pkg/barback/lib/src/asset.dart @@ -5,6 +5,7 @@ library barback.asset; import 'dart:async'; +import 'dart:convert' show Encoding, UTF8; import 'dart:io'; import 'dart:utf'; @@ -37,7 +38,7 @@ abstract class Asset { /// /// If the asset was created from a [String] the original string is always /// returned and [encoding] is ignored. Otherwise, the binary data of the - /// asset is decoded using [encoding], which defaults to [Encoding.UTF_8]. + /// asset is decoded using [encoding], which defaults to [UTF8]. Future readAsString({Encoding encoding}); /// Streams the binary contents of the asset. @@ -54,16 +55,9 @@ class _BinaryAsset extends Asset { : super(id); Future readAsString({Encoding encoding}) { - if (encoding == null) encoding = Encoding.UTF_8; + if (encoding == null) encoding = UTF8; - // TODO(rnystrom): When #6284 is fixed, just use that. Until then, only - // UTF-8 is supported. :( - if (encoding != Encoding.UTF_8) { - throw new UnsupportedError( - "${encoding.name} is not a supported encoding."); - } - - return new Future.value(decodeUtf8(_contents)); + return new Future.value(encoding.decode(_contents)); } Stream> read() => new Future>.value(_contents).asStream(); @@ -104,7 +98,7 @@ class _FileAsset extends Asset { : super(id); Future readAsString({Encoding encoding}) { - if (encoding == null) encoding = Encoding.UTF_8; + if (encoding == null) encoding = UTF8; return _file.readAsString(encoding: encoding); } diff --git a/pkg/barback/test/asset_test.dart b/pkg/barback/test/asset_test.dart index 395dbfdf0254..b26d4629cde8 100644 --- a/pkg/barback/test/asset_test.dart +++ b/pkg/barback/test/asset_test.dart @@ -5,6 +5,7 @@ library barback.test.asset_test; import 'dart:async'; +import 'dart:convert' show Encoding, UTF8, LATIN1; import 'dart:io'; import 'dart:utf'; @@ -108,7 +109,7 @@ main() { test("supports UTF-8", () { var asset = new Asset.fromBytes(id, encodeUtf8("çøñ†éℵ™")); - expect(asset.readAsString(encoding: Encoding.UTF_8), + expect(asset.readAsString(encoding: UTF8), completion(equals("çøñ†éℵ™"))); }); @@ -124,7 +125,7 @@ main() { test("ignores the encoding", () { var asset = new Asset.fromString(id, "contents"); - expect(asset.readAsString(encoding: Encoding.ISO_8859_1), + expect(asset.readAsString(encoding: LATIN1), completion(equals("contents"))); }); }); diff --git a/pkg/barback/test/utils.dart b/pkg/barback/test/utils.dart index 73f4f8b84785..1ee03ff072c8 100644 --- a/pkg/barback/test/utils.dart +++ b/pkg/barback/test/utils.dart @@ -6,6 +6,7 @@ library barback.test.utils; import 'dart:async'; import 'dart:collection'; +import 'dart:convert' show Encoding; import 'dart:io'; import 'package:barback/barback.dart'; diff --git a/pkg/http/lib/src/byte_stream.dart b/pkg/http/lib/src/byte_stream.dart index 20f78747f25d..c75654a28e9b 100644 --- a/pkg/http/lib/src/byte_stream.dart +++ b/pkg/http/lib/src/byte_stream.dart @@ -5,6 +5,7 @@ library byte_stream; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; @@ -30,10 +31,10 @@ class ByteStream extends StreamView> { } /// Collect the data of this stream in a [String], decoded according to - /// [encoding], which defaults to `Encoding.UTF_8`. - Future bytesToString([Encoding encoding=Encoding.UTF_8]) => - toBytes().then((bytes) => decodeString(bytes, encoding)); + /// [encoding], which defaults to `UTF8`. + Future bytesToString([Encoding encoding=UTF8]) => + toBytes().then((bytes) => encoding.decode(bytes)); - Stream toStringStream([Encoding encoding=Encoding.UTF_8]) => - transform(new StringDecoder(encoding)); + Stream toStringStream([Encoding encoding=UTF8]) => + transform(encoding.decoder); } diff --git a/pkg/http/lib/src/multipart_file.dart b/pkg/http/lib/src/multipart_file.dart index 29ef589c024c..19c03bf12ce2 100644 --- a/pkg/http/lib/src/multipart_file.dart +++ b/pkg/http/lib/src/multipart_file.dart @@ -5,6 +5,7 @@ library multipart_file; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:path/path.dart' as path; @@ -70,14 +71,14 @@ class MultipartFile { contentType = contentType == null ? new ContentType("text", "plain") : contentType; var charset = contentType.charset; - var encoding = encodingForCharset(contentType.charset, Encoding.UTF_8); + var encoding = encodingForCharset(contentType.charset, UTF8); // Make a new contentType with ensured charset. contentType = new ContentType(contentType.primaryType, contentType.subType, charset: encoding.name, parameters: contentType.parameters); - return new MultipartFile.fromBytes(field, encodeString(value, encoding), + return new MultipartFile.fromBytes(field, encoding.encode(value), filename: filename, contentType: contentType); } diff --git a/pkg/http/lib/src/request.dart b/pkg/http/lib/src/request.dart index 9f024e6e95fb..8aacbe387877 100644 --- a/pkg/http/lib/src/request.dart +++ b/pkg/http/lib/src/request.dart @@ -4,6 +4,7 @@ library request; +import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; @@ -36,7 +37,7 @@ class Request extends BaseRequest { /// If the request has a `Content-Type` header and that header has a `charset` /// parameter, that parameter's value is used as the encoding. Otherwise, if /// [encoding] has been set manually, that encoding is used. If that hasn't - /// been set either, this defaults to [Encoding.UTF_8]. + /// been set either, this defaults to [UTF8]. /// /// If the `charset` parameter's value is not a known [Encoding], reading this /// will throw a [FormatException]. @@ -83,10 +84,10 @@ class Request extends BaseRequest { /// header, one will be added with the type `text/plain`. Then the `charset` /// parameter of the `Content-Type` header (whether new or pre-existing) will /// be set to [encoding] if it wasn't already set. - String get body => decodeString(bodyBytes, encoding); + String get body => encoding.decode(bodyBytes); set body(String value) { - bodyBytes = encodeString(value, encoding); + bodyBytes = encoding.encode(value); var contentType = _contentType; if (contentType == null) { contentType = new ContentType("text", "plain", charset: encoding.name); @@ -137,7 +138,7 @@ class Request extends BaseRequest { /// Creates a new HTTP request. Request(String method, Uri url) : super(method, url), - _defaultEncoding = Encoding.UTF_8, + _defaultEncoding = UTF8, _bodyBytes = new Uint8List(0); /// Freezes all mutable fields and returns a single-subscription [ByteStream] diff --git a/pkg/http/lib/src/response.dart b/pkg/http/lib/src/response.dart index b7e0a481da81..435027d7e83f 100644 --- a/pkg/http/lib/src/response.dart +++ b/pkg/http/lib/src/response.dart @@ -5,6 +5,7 @@ library response; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; @@ -21,10 +22,10 @@ class Response extends BaseResponse { /// The body of the response as a string. This is converted from [bodyBytes] /// using the `charset` parameter of the `Content-Type` header field, if /// available. If it's unavailable or if the encoding name is unknown, - /// [Encoding.ISO_8859_1] is used by default, as per [RFC 2616][]. + /// [LATIN1] is used by default, as per [RFC 2616][]. /// /// [RFC 2616]: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html - String get body => decodeString(bodyBytes, _encodingForHeaders(headers)); + String get body => _encodingForHeaders(headers).decode(bodyBytes); /// Creates a new HTTP response with a string body. Response( @@ -36,7 +37,7 @@ class Response extends BaseResponse { bool persistentConnection: true, String reasonPhrase}) : this.bytes( - encodeString(body, _encodingForHeaders(headers)), + _encodingForHeaders(headers).encode(body), statusCode, request: request, headers: headers, @@ -80,7 +81,7 @@ class Response extends BaseResponse { } /// Returns the encoding to use for a response with the given headers. This -/// defaults to [Encoding.ISO_8859_1] if the headers don't specify a charset or +/// defaults to [LATIN1] if the headers don't specify a charset or /// if that charset is unknown. Encoding _encodingForHeaders(Map headers) => encodingForCharset(_contentTypeForHeaders(headers).charset); diff --git a/pkg/http/lib/src/utils.dart b/pkg/http/lib/src/utils.dart index 420a3d55441a..cf2096d74dfa 100644 --- a/pkg/http/lib/src/utils.dart +++ b/pkg/http/lib/src/utils.dart @@ -5,9 +5,9 @@ library utils; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; -import 'dart:utf'; import 'byte_stream.dart'; @@ -66,9 +66,9 @@ List split1(String toSplit, String pattern) { /// [charset] is null or if no [Encoding] was found that corresponds to /// [charset]. Encoding encodingForCharset( - String charset, [Encoding fallback = Encoding.ISO_8859_1]) { + String charset, [Encoding fallback = LATIN1]) { if (charset == null) return fallback; - var encoding = Encoding.fromName(charset); + var encoding = Encoding.getByName(charset); return encoding == null ? fallback : encoding; } @@ -77,23 +77,11 @@ Encoding encodingForCharset( /// [FormatException] if no [Encoding] was found that corresponds to [charset]. /// [charset] may not be null. Encoding requiredEncodingForCharset(String charset) { - var encoding = Encoding.fromName(charset); + var encoding = Encoding.getByName(charset); if (encoding != null) return encoding; throw new FormatException('Unsupported encoding "$charset".'); } -/// Converts [bytes] into a [String] according to [encoding]. -String decodeString(List bytes, Encoding encoding) { - // TODO(nweiz): implement this once issue 6284 is fixed. - return decodeUtf8(bytes); -} - -/// Converts [string] into a byte array according to [encoding]. -List encodeString(String string, Encoding encoding) { - // TODO(nweiz): implement this once issue 6284 is fixed. - return encodeUtf8(string); -} - /// A regular expression that matches strings that are composed entirely of /// ASCII-compatible characters. final RegExp _ASCII_ONLY = new RegExp(r"^[\x00-\x7F]+$"); diff --git a/pkg/http/test/request_test.dart b/pkg/http/test/request_test.dart index c9d82cef778f..4761dc066b03 100644 --- a/pkg/http/test/request_test.dart +++ b/pkg/http/test/request_test.dart @@ -4,6 +4,7 @@ library request_test; +import 'dart:convert'; import 'dart:io'; import 'package:http/http.dart' as http; @@ -62,32 +63,32 @@ void main() { group('#encoding', () { test('defaults to utf-8', () { var request = new http.Request('POST', dummyUrl); - expect(request.encoding.name, equals(Encoding.UTF_8.name)); + expect(request.encoding.name, equals(UTF8.name)); }); test('can be set', () { var request = new http.Request('POST', dummyUrl); - request.encoding = Encoding.ISO_8859_1; - expect(request.encoding.name, equals(Encoding.ISO_8859_1.name)); + request.encoding = LATIN1; + expect(request.encoding.name, equals(LATIN1.name)); }); test('is based on the content-type charset if it exists', () { var request = new http.Request('POST', dummyUrl); request.headers[HttpHeaders.CONTENT_TYPE] = 'text/plain; charset=iso-8859-1'; - expect(request.encoding.name, equals(Encoding.ISO_8859_1.name)); + expect(request.encoding.name, equals(LATIN1.name)); }); test('remains the default if the content-type charset is set and unset', () { var request = new http.Request('POST', dummyUrl); - request.encoding = Encoding.ISO_8859_1; + request.encoding = LATIN1; request.headers[HttpHeaders.CONTENT_TYPE] = 'text/plain; charset=utf-8'; - expect(request.encoding.name, equals(Encoding.UTF_8.name)); + expect(request.encoding.name, equals(UTF8.name)); request.headers.remove(HttpHeaders.CONTENT_TYPE); - expect(request.encoding.name, equals(Encoding.ISO_8859_1.name)); + expect(request.encoding.name, equals(LATIN1.name)); }); test('throws an error if the content-type charset is unknown', () { @@ -240,7 +241,7 @@ void main() { test('defaults to empty if only encoding is set', () { var request = new http.Request('POST', dummyUrl); - request.encoding = Encoding.ISO_8859_1; + request.encoding = LATIN1; expect(request.headers[HttpHeaders.CONTENT_TYPE], isNull); }); @@ -255,7 +256,7 @@ void main() { test('is set to application/x-www-form-urlencoded with the given charset ' 'if bodyFields and encoding are set', () { var request = new http.Request('POST', dummyUrl); - request.encoding = Encoding.ISO_8859_1; + request.encoding = LATIN1; request.bodyFields = {'hello': 'world'}; expect(request.headers[HttpHeaders.CONTENT_TYPE], equals('application/x-www-form-urlencoded; charset=iso-8859-1')); @@ -264,7 +265,7 @@ void main() { test('is set to text/plain and the given encoding if body and encoding are ' 'both set', () { var request = new http.Request('POST', dummyUrl); - request.encoding = Encoding.ISO_8859_1; + request.encoding = LATIN1; request.body = 'hello, world'; expect(request.headers[HttpHeaders.CONTENT_TYPE], equals('text/plain; charset=iso-8859-1')); @@ -281,7 +282,7 @@ void main() { test('is modified to include the given encoding if encoding is set', () { var request = new http.Request('POST', dummyUrl); request.headers[HttpHeaders.CONTENT_TYPE] = 'application/json'; - request.encoding = Encoding.ISO_8859_1; + request.encoding = LATIN1; expect(request.headers[HttpHeaders.CONTENT_TYPE], equals('application/json; charset=iso-8859-1')); }); @@ -290,7 +291,7 @@ void main() { var request = new http.Request('POST', dummyUrl); request.headers[HttpHeaders.CONTENT_TYPE] = 'application/json; charset=utf-8'; - request.encoding = Encoding.ISO_8859_1; + request.encoding = LATIN1; expect(request.headers[HttpHeaders.CONTENT_TYPE], equals('application/json; charset=iso-8859-1')); }); @@ -350,8 +351,8 @@ void main() { var request = new http.Request('POST', dummyUrl); request.finalize(); - expect(request.encoding.name, equals(Encoding.UTF_8.name)); - expect(() => request.encoding = Encoding.ASCII, throwsStateError); + expect(request.encoding.name, equals(UTF8.name)); + expect(() => request.encoding = ASCII, throwsStateError); }); test('freezes #bodyBytes', () { diff --git a/pkg/http/test/utils.dart b/pkg/http/test/utils.dart index 9c6e584d4aed..7b42bec00163 100644 --- a/pkg/http/test/utils.dart +++ b/pkg/http/test/utils.dart @@ -5,6 +5,7 @@ library test_utils; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:json' as json; @@ -62,7 +63,7 @@ Future startServer() { if (encodingName != null) { outputEncoding = requiredEncodingForCharset(encodingName); } else { - outputEncoding = Encoding.ASCII; + outputEncoding = ASCII; } response.headers.contentType = @@ -76,7 +77,7 @@ Future startServer() { } else if (request.headers.contentType.charset != null) { var encoding = requiredEncodingForCharset( request.headers.contentType.charset); - requestBody = decodeString(requestBodyBytes, encoding); + requestBody = encoding.decode(requestBodyBytes); } else { requestBody = requestBodyBytes; } diff --git a/pkg/http_server/lib/http_server.dart b/pkg/http_server/lib/http_server.dart index 123ff7ca4d20..71763309e069 100644 --- a/pkg/http_server/lib/http_server.dart +++ b/pkg/http_server/lib/http_server.dart @@ -5,6 +5,7 @@ library http_server; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:json' as JSON; diff --git a/pkg/http_server/lib/src/http_body.dart b/pkg/http_server/lib/src/http_body.dart index 422f9dcb0e65..f05d5b22e043 100644 --- a/pkg/http_server/lib/src/http_body.dart +++ b/pkg/http_server/lib/src/http_body.dart @@ -103,7 +103,7 @@ class HttpBodyHandler * `multipart/form-data` content correctly. See the class comment * for more information on `multipart/form-data`. */ - HttpBodyHandler({Encoding defaultEncoding: Encoding.UTF_8}) + HttpBodyHandler({Encoding defaultEncoding: UTF8}) : _transformer = new _HttpBodyHandlerTransformer(defaultEncoding); /** @@ -114,7 +114,7 @@ class HttpBodyHandler */ static Future processRequest( HttpRequest request, - {Encoding defaultEncoding: Encoding.UTF_8}) { + {Encoding defaultEncoding: UTF8}) { return _HttpBodyHandler.processRequest(request, defaultEncoding); } @@ -125,7 +125,7 @@ class HttpBodyHandler */ static Future processResponse( HttpClientResponse response, - {Encoding defaultEncoding: Encoding.UTF_8}) { + {Encoding defaultEncoding: UTF8}) { return _HttpBodyHandler.processResponse(response, defaultEncoding); } diff --git a/pkg/http_server/lib/src/http_body_impl.dart b/pkg/http_server/lib/src/http_body_impl.dart index 5d52874b1bd4..c48db0378a9c 100644 --- a/pkg/http_server/lib/src/http_body_impl.dart +++ b/pkg/http_server/lib/src/http_body_impl.dart @@ -53,10 +53,10 @@ class _HttpBodyHandler { Future asText(Encoding defaultEncoding) { var encoding; var charset = contentType.charset; - if (charset != null) encoding = Encoding.fromName(charset); + if (charset != null) encoding = Encoding.getByName(charset); if (encoding == null) encoding = defaultEncoding; return stream - .transform(new StringDecoder(encoding)) + .transform(encoding.decoder) .fold(new StringBuffer(), (buffer, data) => buffer..write(data)) .then((buffer) => new _HttpBody(contentType, "text", @@ -107,21 +107,21 @@ class _HttpBodyHandler { switch (contentType.primaryType) { case "text": - return asText(Encoding.ASCII); + return asText(ASCII); case "application": switch (contentType.subType) { case "json": - return asText(Encoding.UTF_8) + return asText(UTF8) .then((body) => new _HttpBody(contentType, "json", JSON.parse(body.body))); case "x-www-form-urlencoded": - return asText(Encoding.ASCII) + return asText(ASCII) .then((body) { var map = Uri.splitQueryString(body.body, - decode: (s) => _decodeString(s, defaultEncoding)); + decode: (s) => defaultEncoding.decode(s)); var result = {}; for (var key in map.keys) { result[key] = map[key]; @@ -150,24 +150,6 @@ class _HttpBodyHandler { return asBinary(); } - - // Utility function to synchronously decode a list of bytes. - static String _decodeString(List bytes, - [Encoding encoding = Encoding.UTF_8]) { - if (bytes.length == 0) return ""; - var string; - var error; - var controller = new StreamController(sync: true); - controller.stream - .transform(new StringDecoder(encoding)) - .listen((data) => string = data, - onError: (e) => error = e); - controller.add(bytes); - controller.close(); - if (error != null) throw error; - assert(string != null); - return string; - } } class _HttpBodyFileUpload implements HttpBodyFileUpload { diff --git a/pkg/http_server/lib/src/http_multipart_form_data_impl.dart b/pkg/http_server/lib/src/http_multipart_form_data_impl.dart index 8b313aa1bdc1..7956c0f0bb34 100644 --- a/pkg/http_server/lib/src/http_multipart_form_data_impl.dart +++ b/pkg/http_server/lib/src/http_multipart_form_data_impl.dart @@ -34,11 +34,11 @@ class _HttpMultipartFormData extends Stream implements HttpMultipartFormData { StringBuffer buffer = new StringBuffer(); Encoding encoding; if (contentType != null) { - encoding = Encoding.fromName(contentType.charset); + encoding = Encoding.getByName(contentType.charset); } - if (encoding == null) encoding = Encoding.ISO_8859_1; + if (encoding == null) encoding = LATIN1; _stream = _stream - .transform(new StringDecoder(encoding)) + .transform(encoding.decoder) .expand((data) { buffer.write(data); var out = _decodeHttpEntityString(buffer.toString()); diff --git a/pkg/http_server/test/http_body_test.dart b/pkg/http_server/test/http_body_test.dart index 725480b192ac..7f510e73426a 100644 --- a/pkg/http_server/test/http_body_test.dart +++ b/pkg/http_server/test/http_body_test.dart @@ -3,7 +3,7 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:io'; -import 'dart:utf'; +import 'dart:convert'; import 'package:http_server/http_server.dart'; import 'package:unittest/unittest.dart'; @@ -66,11 +66,8 @@ void testHttpClientResponseBody() { "body", "text"); test("text/plain; charset=utf-8", [42], "*", "text"); - test("text/plain; charset=us-ascii", [142], "?", "text"); - test("text/plain; charset=utf-8", - [142], - new String.fromCharCodes([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]), - "text"); + test("text/plain; charset=us-ascii", [142], null, "text", true); + test("text/plain; charset=utf-8", [142], null, "text", true); test("application/json", '{"val": 5}'.codeUnits, @@ -89,7 +86,7 @@ void testHttpServerRequestBody() { dynamic expectedBody, String type, {bool shouldFail: false, - Encoding defaultEncoding: Encoding.UTF_8}) { + Encoding defaultEncoding: UTF8}) { HttpServer.bind("127.0.0.1", 0).then((server) { server.transform(new HttpBodyHandler(defaultEncoding: defaultEncoding)) .listen((body) { @@ -169,11 +166,8 @@ void testHttpServerRequestBody() { "body", "text"); test("text/plain; charset=utf-8", [42], "*", "text"); - test("text/plain; charset=us-ascii", [142], "?", "text"); - test("text/plain; charset=utf-8", - [142], - new String.fromCharCodes([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]), - "text"); + test("text/plain; charset=us-ascii", [142], null, "text", shouldFail: true); + test("text/plain; charset=utf-8", [142], null, "text", shouldFail: true); test("application/json", '{"val": 5}'.codeUnits, @@ -250,23 +244,27 @@ File content\r test('application/x-www-form-urlencoded', 'a=%F8+%26%23548%3B'.codeUnits, - { 'a' : '\u{FFFD} Ȥ' }, - "form"); + null, + "form", + shouldFail: true); test('application/x-www-form-urlencoded', 'a=%C0%A0'.codeUnits, - { 'a' : '\u{FFFD}' }, - "form"); + null, + "form", + shouldFail: true); test('application/x-www-form-urlencoded', 'a=x%A0x'.codeUnits, - { 'a' : 'x\u{FFFD}x' }, - "form"); + null, + "form", + shouldFail: true); test('application/x-www-form-urlencoded', 'a=x%C0x'.codeUnits, - { 'a' : 'x\u{FFFD}x' }, - "form"); + null, + "form", + shouldFail: true); test('application/x-www-form-urlencoded', 'a=%C3%B8+%C8%A4'.codeUnits, @@ -277,31 +275,31 @@ File content\r 'a=%F8+%26%23548%3B'.codeUnits, { 'a' : 'ø Ȥ' }, "form", - defaultEncoding: Encoding.ISO_8859_1); + defaultEncoding: LATIN1); test('application/x-www-form-urlencoded', 'name=%26'.codeUnits, { 'name' : '&' }, "form", - defaultEncoding: Encoding.ISO_8859_1); + defaultEncoding: LATIN1); test('application/x-www-form-urlencoded', 'name=%F8%26'.codeUnits, { 'name' : 'ø&' }, "form", - defaultEncoding: Encoding.ISO_8859_1); + defaultEncoding: LATIN1); test('application/x-www-form-urlencoded', 'name=%26%3B'.codeUnits, { 'name' : '&;' }, "form", - defaultEncoding: Encoding.ISO_8859_1); + defaultEncoding: LATIN1); test('application/x-www-form-urlencoded', 'name=%26%23548%3B%26%23548%3B'.codeUnits, { 'name' : 'ȤȤ' }, "form", - defaultEncoding: Encoding.ISO_8859_1); + defaultEncoding: LATIN1); test('application/x-www-form-urlencoded', 'name=%26'.codeUnits, diff --git a/pkg/http_server/test/utils.dart b/pkg/http_server/test/utils.dart index 6a46696cff4c..c9e39c7c61bd 100644 --- a/pkg/http_server/test/utils.dart +++ b/pkg/http_server/test/utils.dart @@ -5,6 +5,7 @@ library utils; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import "package:path/path.dart"; @@ -49,7 +50,7 @@ Future getHeaders(int port, String path) { Future getAsString(int port, String path) { return new HttpClient().get('localhost', port, path) .then((request) => request.close()) - .then((response) => StringDecoder.decode(response)); + .then((response) => UTF8.decodeStream(response)); } diff --git a/pkg/scheduled_test/lib/scheduled_process.dart b/pkg/scheduled_test/lib/scheduled_process.dart index 27ebc17ef285..4768c684fefe 100644 --- a/pkg/scheduled_test/lib/scheduled_process.dart +++ b/pkg/scheduled_test/lib/scheduled_process.dart @@ -82,7 +82,7 @@ class ScheduledProcess { /// may be a [List] containing a mix of strings and [Future]s. ScheduledProcess.start(executable, arguments, {workingDirectory, environment, String description, - Encoding encoding: Encoding.UTF_8}) + Encoding encoding: UTF8}) : _encoding = encoding, _explicitDescription = description != null, _description = description { @@ -152,7 +152,7 @@ class ScheduledProcess { arguments, workingDirectory: workingDirectory, environment: environment).then((process) { - process.stdin.encoding = Encoding.UTF_8; + process.stdin.encoding = UTF8; return process; }); }); @@ -203,7 +203,7 @@ class ScheduledProcess { currentSchedule.heartbeat(); return chunk; }) - .transform(new StringDecoder(_encoding)) + .transform(_encoding.decoder) .transform(new LineSplitter())); } diff --git a/pkg/scheduled_test/test/scheduled_process_test.dart b/pkg/scheduled_test/test/scheduled_process_test.dart index 33cfa7c7f7bd..5aaae605fedc 100644 --- a/pkg/scheduled_test/test/scheduled_process_test.dart +++ b/pkg/scheduled_test/test/scheduled_process_test.dart @@ -372,7 +372,7 @@ ScheduledProcess startDartProcess(String script) { import 'dart:io'; var stdinLines = stdin - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); void main() { diff --git a/runtime/bin/process_patch.dart b/runtime/bin/process_patch.dart index 62489ca65205..441e75b973cc 100644 --- a/runtime/bin/process_patch.dart +++ b/runtime/bin/process_patch.dart @@ -38,8 +38,8 @@ patch class Process { Map environment, bool includeParentEnvironment: true, bool runInShell: false, - Encoding stdoutEncoding: Encoding.SYSTEM, - Encoding stderrEncoding: Encoding.SYSTEM}) { + Encoding stdoutEncoding: SYSTEM_ENCODING, + Encoding stderrEncoding: SYSTEM_ENCODING}) { return _runNonInteractiveProcess(executable, arguments, workingDirectory, @@ -57,8 +57,8 @@ patch class Process { Map environment, bool includeParentEnvironment: true, bool runInShell: false, - Encoding stdoutEncoding: Encoding.SYSTEM, - Encoding stderrEncoding: Encoding.SYSTEM}) { + Encoding stdoutEncoding: SYSTEM_ENCODING, + Encoding stderrEncoding: SYSTEM_ENCODING}) { return _runNonInteractiveProcessSync(executable, arguments, workingDirectory, @@ -324,7 +324,7 @@ class _ProcessImpl extends NativeFieldWrapperClass1 implements Process { getOutput(output, encoding) { if (stderrEncoding == null) return output; - return _decodeString(output, encoding); + return encoding.decode(output); } return new _ProcessResult( @@ -424,7 +424,7 @@ Future _runNonInteractiveProcess(String path, .then((builder) => builder.takeBytes()); } else { return stream - .transform(new StringDecoder(encoding)) + .transform(encoding.decoder) .fold( new StringBuffer(), (buf, data) { diff --git a/runtime/vm/exceptions.cc b/runtime/vm/exceptions.cc index 62594515e5f0..a57a46a5db21 100644 --- a/runtime/vm/exceptions.cc +++ b/runtime/vm/exceptions.cc @@ -412,21 +412,13 @@ static void ThrowExceptionHelper(const Instance& incoming_exception, // TODO(5411263): At some point we can optimize by figuring out if a // stack trace is needed based on whether the catch code specifies a // stack trace object or there is a rethrow in the catch clause. - if (pc_offset_array.Length() != 0) { - if (existing_stacktrace.IsNull()) { + if (existing_stacktrace.IsNull()) { stacktrace = Stacktrace::New(func_array, code_array, pc_offset_array); - } else { - stacktrace ^= existing_stacktrace.raw(); - stacktrace.Append(func_array, code_array, pc_offset_array); - // Since we are re throwing and appending to the existing stack trace - // we clear out the catch trace collected in the existing stack trace - // as that trace will not be valid anymore. - stacktrace.SetCatchStacktrace(Object::empty_array(), - Object::empty_array(), - Object::empty_array()); - } } else { stacktrace ^= existing_stacktrace.raw(); + if (pc_offset_array.Length() != 0) { + stacktrace.Append(func_array, code_array, pc_offset_array); + } // Since we are re throwing and appending to the existing stack trace // we clear out the catch trace collected in the existing stack trace // as that trace will not be valid anymore. diff --git a/runtime/vm/flow_graph_type_propagator.cc b/runtime/vm/flow_graph_type_propagator.cc index 2ea65bac1ed8..ba39000ec67e 100644 --- a/runtime/vm/flow_graph_type_propagator.cc +++ b/runtime/vm/flow_graph_type_propagator.cc @@ -491,8 +491,7 @@ static bool IsKnownPrivateClass(const Class& type_class) { intptr_t CompileType::ToNullableCid() { if (cid_ == kIllegalCid) { if (type_ == NULL) { - // Type propagation is turned off. - ASSERT(!FLAG_propagate_types); + // Type propagation is turned off or has not yet run. return kDynamicCid; } else if (type_->IsMalformed()) { cid_ = kDynamicCid; diff --git a/runtime/vm/intrinsifier_arm.cc b/runtime/vm/intrinsifier_arm.cc index 312912eb58d1..1a528f034416 100644 --- a/runtime/vm/intrinsifier_arm.cc +++ b/runtime/vm/intrinsifier_arm.cc @@ -1480,7 +1480,7 @@ bool Intrinsifier::String_getIsEmpty(Assembler* assembler) { __ LoadObject(R0, Bool::True(), EQ); __ LoadObject(R0, Bool::False(), NE); __ Ret(); - return true; + return false; } diff --git a/runtime/vm/intrinsifier_ia32.cc b/runtime/vm/intrinsifier_ia32.cc index 07932b01e196..aac288d2408d 100644 --- a/runtime/vm/intrinsifier_ia32.cc +++ b/runtime/vm/intrinsifier_ia32.cc @@ -1550,7 +1550,7 @@ bool Intrinsifier::String_getIsEmpty(Assembler* assembler) { __ Bind(&is_true); __ LoadObject(EAX, Bool::True()); __ ret(); - return true; + return false; } diff --git a/runtime/vm/intrinsifier_mips.cc b/runtime/vm/intrinsifier_mips.cc index 517e89b33c8e..81dacc74fedf 100644 --- a/runtime/vm/intrinsifier_mips.cc +++ b/runtime/vm/intrinsifier_mips.cc @@ -1558,7 +1558,7 @@ bool Intrinsifier::String_getIsEmpty(Assembler* assembler) { __ Bind(&is_true); __ LoadObject(V0, Bool::True()); __ Ret(); - return true; + return false; } diff --git a/runtime/vm/intrinsifier_x64.cc b/runtime/vm/intrinsifier_x64.cc index aa3ebffd8c1b..f6d70b6d5a2f 100644 --- a/runtime/vm/intrinsifier_x64.cc +++ b/runtime/vm/intrinsifier_x64.cc @@ -1466,7 +1466,7 @@ bool Intrinsifier::String_getIsEmpty(Assembler* assembler) { __ Bind(&is_true); __ LoadObject(RAX, Bool::True()); __ ret(); - return true; + return false; } diff --git a/runtime/vm/parser.cc b/runtime/vm/parser.cc index 9815cea1b52f..cdbb608d1729 100644 --- a/runtime/vm/parser.cc +++ b/runtime/vm/parser.cc @@ -762,12 +762,27 @@ void Parser::ParseClass(const Class& cls) { } +static bool IsInvisible(const Function& func) { + if (!Library::IsPrivate(String::Handle(func.name()))) return false; + // Check for private function in the core libraries. + const Class& cls = Class::Handle(func.Owner()); + const Library& library = Library::Handle(cls.library()); + if (library.raw() == Library::CoreLibrary()) return true; + if (library.raw() == Library::CollectionLibrary()) return true; + if (library.raw() == Library::TypedDataLibrary()) return true; + if (library.raw() == Library::MathLibrary()) return true; + return false; +} + + void Parser::ParseFunction(ParsedFunction* parsed_function) { TimerScope timer(FLAG_compiler_stats, &CompilerStats::parser_timer); Isolate* isolate = Isolate::Current(); ASSERT(isolate->long_jump_base()->IsSafeToJump()); ASSERT(parsed_function != NULL); const Function& func = parsed_function->function(); + // Mark private core library functions as invisible by default. + if (IsInvisible(func)) func.set_is_visible(false); const Script& script = Script::Handle(isolate, func.script()); Parser parser(script, parsed_function, func.token_pos()); SequenceNode* node_sequence = NULL; diff --git a/sdk/lib/_internal/lib/io_patch.dart b/sdk/lib/_internal/lib/io_patch.dart index 80725b894e13..0bd68ac8d0d0 100644 --- a/sdk/lib/_internal/lib/io_patch.dart +++ b/sdk/lib/_internal/lib/io_patch.dart @@ -208,8 +208,8 @@ patch class Process { Map environment, bool includeParentEnvironment: true, bool runInShell: false, - Encoding stdoutEncoding: Encoding.SYSTEM, - Encoding stderrEncoding: Encoding.SYSTEM}) { + Encoding stdoutEncoding: SYSTEM_ENCODING, + Encoding stderrEncoding: SYSTEM_ENCODING}) { throw new UnsupportedError("Process.run"); } @@ -220,8 +220,8 @@ patch class Process { Map environment, bool includeParentEnvironment: true, bool runInShell: false, - Encoding stdoutEncoding: Encoding.SYSTEM, - Encoding stderrEncoding: Encoding.SYSTEM}) { + Encoding stdoutEncoding: SYSTEM_ENCODING, + Encoding stderrEncoding: SYSTEM_ENCODING}) { throw new UnsupportedError("Process.runSync"); } } diff --git a/sdk/lib/_internal/pub/lib/src/io.dart b/sdk/lib/_internal/pub/lib/src/io.dart index 5847fedfed3b..0bd5c1420183 100644 --- a/sdk/lib/_internal/pub/lib/src/io.dart +++ b/sdk/lib/_internal/pub/lib/src/io.dart @@ -7,6 +7,7 @@ library pub.io; import 'dart:async'; import 'dart:collection'; +import 'dart:convert'; import 'dart:io'; import 'package:path/path.dart' as path; @@ -136,7 +137,7 @@ String resolveLink(String link) { /// Reads the contents of the text file [file]. String readTextFile(String file) => - new File(file).readAsStringSync(encoding: Encoding.UTF_8); + new File(file).readAsStringSync(encoding: UTF8); /// Reads the contents of the binary file [file]. List readBinaryFile(String file) { diff --git a/sdk/lib/_internal/pub/lib/src/safe_http_server.dart b/sdk/lib/_internal/pub/lib/src/safe_http_server.dart index 634a76fc2017..f5f57cf651a8 100644 --- a/sdk/lib/_internal/pub/lib/src/safe_http_server.dart +++ b/sdk/lib/_internal/pub/lib/src/safe_http_server.dart @@ -5,6 +5,7 @@ library pub.safe_http_server; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; // TODO(nweiz): remove this when issue 9140 is fixed. diff --git a/sdk/lib/_internal/pub/test/test_pub.dart b/sdk/lib/_internal/pub/test/test_pub.dart index f1358a9e9e57..81e74e8bd2da 100644 --- a/sdk/lib/_internal/pub/test/test_pub.dart +++ b/sdk/lib/_internal/pub/test/test_pub.dart @@ -9,6 +9,7 @@ library test_pub; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:json' as json; import 'dart:math'; @@ -501,7 +502,7 @@ class PubProcess extends ScheduledProcess { PubProcess.start(executable, arguments, {workingDirectory, environment, String description, - Encoding encoding: Encoding.UTF_8}) + Encoding encoding: UTF8}) : super.start(executable, arguments, workingDirectory: workingDirectory, environment: environment, diff --git a/sdk/lib/convert/ascii.dart b/sdk/lib/convert/ascii.dart index bea9680aea20..56d6c89b6938 100644 --- a/sdk/lib/convert/ascii.dart +++ b/sdk/lib/convert/ascii.dart @@ -24,7 +24,7 @@ const int _ASCII_MASK = 0x7F; * An [AsciiCodec] allows encoding strings as ASCII bytes * and decoding ASCII bytes to strings. */ -class AsciiCodec extends _Encoding { +class AsciiCodec extends Encoding { final bool _allowInvalid; /** * Instantiates a new [AsciiCodec]. @@ -39,6 +39,8 @@ class AsciiCodec extends _Encoding { */ const AsciiCodec({bool allowInvalid: false}) : _allowInvalid = allowInvalid; + String get name => "us-ascii"; + /** * Decodes the ASCII [bytes] (a list of unsigned 7-bit integers) to the * corresponding string. diff --git a/sdk/lib/convert/encoding.dart b/sdk/lib/convert/encoding.dart index 0f6126c25130..cc67f6f176c5 100644 --- a/sdk/lib/convert/encoding.dart +++ b/sdk/lib/convert/encoding.dart @@ -7,13 +7,70 @@ part of dart.convert; /** * Open-ended Encoding enum. */ -// TODO(floitsch): dart:io already has an Encoding class. If we can't -// consolitate them, we need to remove `Encoding` here. -abstract class _Encoding extends Codec> { - const _Encoding(); +abstract class Encoding extends Codec> { + const Encoding(); - // TODO(floitsch): should we introduce a StringToByteEncoder and - // a ByteToStringDecoder so that we have better typing? -} + Future decodeStream(Stream> byteStream) { + return byteStream + .transform(decoder) + .fold(new StringBuffer(), (buffer, string) => buffer..write(string)) + .then((buffer) => buffer.toString()); + } + + /** + * Name of the encoding. + * + * If the encoding is standardized, this is the lower-case version of one of + * the IANA official names for the character set (see + * http://www.iana.org/assignments/character-sets/character-sets.xml) + */ + String get name; + + // All aliases (in lowercase) of supported encoding from + // http://www.iana.org/assignments/character-sets/character-sets.xml. + static Map _nameToEncoding = { + // ISO_8859-1:1987. + "iso_8859-1:1987": LATIN1, + "iso-ir-100": LATIN1, + "iso_8859-1": LATIN1, + "iso-8859-1": LATIN1, + "latin1": LATIN1, + "l1": LATIN1, + "ibm819": LATIN1, + "cp819": LATIN1, + "csisolatin1": LATIN1, -// TODO(floitsch): add other encodings, like ASCII and ISO_8859_1. + // US-ASCII. + "iso-ir-6": ASCII, + "ansi_x3.4-1968": ASCII, + "ansi_x3.4-1986": ASCII, + "iso_646.irv:1991": ASCII, + "iso646-us": ASCII, + "us-ascii": ASCII, + "us": ASCII, + "ibm367": ASCII, + "cp367": ASCII, + "csascii": ASCII, + "ascii": ASCII, // This is not in the IANA official names. + + // UTF-8. + "csutf8": UTF8, + "utf-8": UTF8 + }; + + /** + * Gets an [Encoding] object from the name of the character set + * name. The names used are the IANA official names for the + * character set (see + * http://www.iana.org/assignments/character-sets/character-sets.xml). + * + * The [name] passed is case insensitive. + * + * If character set is not supported [:null:] is returned. + */ + static Encoding getByName(String name) { + if (name == null) return null; + name = name.toLowerCase(); + return _nameToEncoding[name]; + } +} diff --git a/sdk/lib/convert/latin1.dart b/sdk/lib/convert/latin1.dart index 3e9232f942b0..2fca67cc64f3 100644 --- a/sdk/lib/convert/latin1.dart +++ b/sdk/lib/convert/latin1.dart @@ -24,7 +24,7 @@ const int _LATIN1_MASK = 0xFF; * A [LatinCodec] encodes strings to ISO Latin-1 (aka ISO-8859-1) bytes * and decodes Latin-1 bytes to strings. */ -class Latin1Codec extends _Encoding { +class Latin1Codec extends Encoding { final bool _allowInvalid; /** * Instantiates a new [Latin1Codec]. @@ -38,6 +38,8 @@ class Latin1Codec extends _Encoding { */ const Latin1Codec({bool allowInvalid: false}) : _allowInvalid = allowInvalid; + String get name => "iso-8859-1"; + /** * Decodes the Latin-1 [bytes] (a list of unsigned 8-bit integers) to the * corresponding string. diff --git a/sdk/lib/convert/utf.dart b/sdk/lib/convert/utf.dart index 37f233525293..6abaa0519b95 100644 --- a/sdk/lib/convert/utf.dart +++ b/sdk/lib/convert/utf.dart @@ -4,6 +4,9 @@ part of dart.convert; +/** The Unicode Replacement character `U+FFFD` (�). */ +const UNICODE_REPLACEMENT_CHARACTER_RUNE = 0xFFFD; + /** * An instance of the default implementation of the [Utf8Codec]. * @@ -22,7 +25,7 @@ const UTF8 = const Utf8Codec(); * A [Utf8Codec] encodes strings to utf-8 code units (bytes) and decodes * UTF-8 code units to strings. */ -class Utf8Codec extends _Encoding { +class Utf8Codec extends Encoding { final bool _allowMalformed; /** @@ -39,6 +42,8 @@ class Utf8Codec extends _Encoding { const Utf8Codec({ bool allowMalformed: false }) : _allowMalformed = allowMalformed; + String get name => "utf-8"; + /** * Decodes the UTF-8 [codeUnits] (a list of unsigned 8-bit integers) to the * corresponding string. @@ -66,6 +71,9 @@ class Utf8Codec extends _Encoding { * unsigned 8-bit integers). */ class Utf8Encoder extends Converter> { + + const Utf8Encoder(); + /** * Converts [string] to its UTF-8 code units (a list of * unsigned 8-bit integers). @@ -289,7 +297,7 @@ class Utf8Decoder extends Converter, String> { * sequences with the Unicode Replacement character `U+FFFD` (�). Otherwise * it throws a [FormatException]. */ - Utf8Decoder({ bool allowMalformed: false }) + const Utf8Decoder({ bool allowMalformed: false }) : this._allowMalformed = allowMalformed; /** diff --git a/sdk/lib/io/file.dart b/sdk/lib/io/file.dart index 5cbff1cd663e..ffcadc72bfce 100644 --- a/sdk/lib/io/file.dart +++ b/sdk/lib/io/file.dart @@ -204,7 +204,7 @@ abstract class File implements FileSystemEntity { * [IOSink] has been created. */ IOSink openWrite({FileMode mode: FileMode.WRITE, - Encoding encoding: Encoding.UTF_8}); + Encoding encoding: UTF8}); /** * Read the entire file contents as a list of bytes. Returns a @@ -227,7 +227,7 @@ abstract class File implements FileSystemEntity { * Returns a [:Future:] that completes with the string once * the file contents has been read. */ - Future readAsString({Encoding encoding: Encoding.UTF_8}); + Future readAsString({Encoding encoding: UTF8}); /** * Synchronously read the entire file contents as a string using the @@ -235,7 +235,7 @@ abstract class File implements FileSystemEntity { * * Throws a [FileException] if the operation fails. */ - String readAsStringSync({Encoding encoding: Encoding.UTF_8}); + String readAsStringSync({Encoding encoding: UTF8}); /** * Read the entire file contents as lines of text using the given @@ -244,7 +244,7 @@ abstract class File implements FileSystemEntity { * Returns a [:Future>:] that completes with the lines * once the file contents has been read. */ - Future> readAsLines({Encoding encoding: Encoding.UTF_8}); + Future> readAsLines({Encoding encoding: UTF8}); /** * Synchronously read the entire file contents as lines of text @@ -252,7 +252,7 @@ abstract class File implements FileSystemEntity { * * Throws a [FileException] if the operation fails. */ - List readAsLinesSync({Encoding encoding: Encoding.UTF_8}); + List readAsLinesSync({Encoding encoding: UTF8}); /** * Write a list of bytes to a file. @@ -293,7 +293,7 @@ abstract class File implements FileSystemEntity { */ Future writeAsString(String contents, {FileMode mode: FileMode.WRITE, - Encoding encoding: Encoding.UTF_8}); + Encoding encoding: UTF8}); /** * Synchronously write a string to a file. @@ -310,7 +310,7 @@ abstract class File implements FileSystemEntity { */ void writeAsStringSync(String contents, {FileMode mode: FileMode.WRITE, - Encoding encoding: Encoding.UTF_8}); + Encoding encoding: UTF8}); /** * Get the path of the file. @@ -428,7 +428,7 @@ abstract class RandomAccessFile { * RandomAccessFile when the write completes. */ Future writeString(String string, - {Encoding encoding: Encoding.UTF_8}); + {Encoding encoding: UTF8}); /** * Synchronously writes a single string to the file using the given @@ -437,7 +437,7 @@ abstract class RandomAccessFile { * Throws a [FileException] if the operation fails. */ void writeStringSync(String string, - {Encoding encoding: Encoding.UTF_8}); + {Encoding encoding: UTF8}); /** * Gets the current byte position in the file. Returns a diff --git a/sdk/lib/io/file_impl.dart b/sdk/lib/io/file_impl.dart index 1dd2b476fbf0..3ce6ee1e2f66 100644 --- a/sdk/lib/io/file_impl.dart +++ b/sdk/lib/io/file_impl.dart @@ -466,7 +466,7 @@ class _File implements File { } IOSink openWrite({FileMode mode: FileMode.WRITE, - Encoding encoding: Encoding.UTF_8}) { + Encoding encoding: UTF8}) { if (mode != FileMode.WRITE && mode != FileMode.APPEND) { throw new ArgumentError( @@ -503,16 +503,16 @@ class _File implements File { return builder.takeBytes(); } - Future readAsString({Encoding encoding: Encoding.UTF_8}) { + Future readAsString({Encoding encoding: UTF8}) { _ensureFileService(); return readAsBytes().then((bytes) { - return _decodeString(bytes, encoding); + return encoding.decode(bytes); }); } - String readAsStringSync({Encoding encoding: Encoding.UTF_8}) { + String readAsStringSync({Encoding encoding: UTF8}) { List bytes = readAsBytesSync(); - return _decodeString(bytes, encoding); + return encoding.decode(bytes); } static List _decodeLines(List bytes, Encoding encoding) { @@ -520,7 +520,7 @@ class _File implements File { var list = []; var controller = new StreamController(sync: true); controller.stream - .transform(new StringDecoder(encoding)) + .transform(encoding.decoder) .transform(new LineSplitter()) .listen((line) => list.add(line)); controller.add(bytes); @@ -528,14 +528,14 @@ class _File implements File { return list; } - Future> readAsLines({Encoding encoding: Encoding.UTF_8}) { + Future> readAsLines({Encoding encoding: UTF8}) { _ensureFileService(); return readAsBytes().then((bytes) { return _decodeLines(bytes, encoding); }); } - List readAsLinesSync({Encoding encoding: Encoding.UTF_8}) { + List readAsLinesSync({Encoding encoding: UTF8}) { return _decodeLines(readAsBytesSync(), encoding); } @@ -559,9 +559,9 @@ class _File implements File { Future writeAsString(String contents, {FileMode mode: FileMode.WRITE, - Encoding encoding: Encoding.UTF_8}) { + Encoding encoding: UTF8}) { try { - return writeAsBytes(_encodeString(contents, encoding), mode: mode); + return writeAsBytes(encoding.encode(contents), mode: mode); } catch (e) { return new Future.error(e); } @@ -569,8 +569,8 @@ class _File implements File { void writeAsStringSync(String contents, {FileMode mode: FileMode.WRITE, - Encoding encoding: Encoding.UTF_8}) { - writeAsBytesSync(_encodeString(contents, encoding), mode: mode); + Encoding encoding: UTF8}) { + writeAsBytesSync(encoding.encode(contents), mode: mode); } String toString() => "File: '$path'"; @@ -824,19 +824,19 @@ class _RandomAccessFile implements RandomAccessFile { } Future writeString(String string, - {Encoding encoding: Encoding.UTF_8}) { + {Encoding encoding: UTF8}) { if (encoding is! Encoding) { throw new ArgumentError(encoding); } - var data = _encodeString(string, encoding); + var data = encoding.encode(string); return writeFrom(data, 0, data.length); } - void writeStringSync(String string, {Encoding encoding: Encoding.UTF_8}) { + void writeStringSync(String string, {Encoding encoding: UTF8}) { if (encoding is! Encoding) { throw new ArgumentError(encoding); } - var data = _encodeString(string, encoding); + var data = encoding.encode(string); writeFromSync(data, 0, data.length); } diff --git a/sdk/lib/io/http_impl.dart b/sdk/lib/io/http_impl.dart index f7b904122793..a7cac9fdb064 100644 --- a/sdk/lib/io/http_impl.dart +++ b/sdk/lib/io/http_impl.dart @@ -410,7 +410,7 @@ abstract class _HttpOutboundMessage implements IOSink { String protocolVersion, _HttpOutgoing outgoing) : _outgoing = outgoing, - _headersSink = new IOSink(outgoing, encoding: Encoding.ASCII), + _headersSink = new IOSink(outgoing, encoding: ASCII), headers = new _HttpHeaders(protocolVersion) { _dataSink = new IOSink(new _HttpOutboundConsumer(this)); } @@ -432,7 +432,7 @@ abstract class _HttpOutboundMessage implements IOSink { } else { charset = "iso-8859-1"; } - return Encoding.fromName(charset); + return Encoding.getByName(charset); } void set encoding(Encoding value) { @@ -1222,7 +1222,7 @@ class _HttpClientConnection { // If the proxy configuration contains user information use that // for proxy basic authorization. String auth = _CryptoUtils.bytesToBase64( - _encodeString("${proxy.username}:${proxy.password}")); + UTF8.encode("${proxy.username}:${proxy.password}")); request.headers.set(HttpHeaders.PROXY_AUTHORIZATION, "Basic $auth"); } else if (!proxy.isDirect && _httpClient._proxyCredentials.length > 0) { proxyCreds = _httpClient._findProxyCredentials(proxy); @@ -1234,7 +1234,7 @@ class _HttpClientConnection { // If the URL contains user information use that for basic // authorization. String auth = - _CryptoUtils.bytesToBase64(_encodeString(uri.userInfo)); + _CryptoUtils.bytesToBase64(UTF8.encode(uri.userInfo)); request.headers.set(HttpHeaders.AUTHORIZATION, "Basic $auth"); } else { // Look for credentials. @@ -1345,7 +1345,7 @@ class _HttpClientConnection { // If the proxy configuration contains user information use that // for proxy basic authorization. String auth = _CryptoUtils.bytesToBase64( - _encodeString("${proxy.username}:${proxy.password}")); + UTF8.encode("${proxy.username}:${proxy.password}")); request.headers.set(HttpHeaders.PROXY_AUTHORIZATION, "Basic $auth"); } return request.close() @@ -2270,11 +2270,11 @@ abstract class _Credentials { // now always use UTF-8 encoding. _HttpClientDigestCredentials creds = credentials; var hasher = new _MD5(); - hasher.add(_encodeString(creds.username)); + hasher.add(UTF8.encode(creds.username)); hasher.add([_CharCode.COLON]); hasher.add(realm.codeUnits); hasher.add([_CharCode.COLON]); - hasher.add(_encodeString(creds.password)); + hasher.add(UTF8.encode(creds.password)); ha1 = _CryptoUtils.bytesToHex(hasher.close()); } } @@ -2363,7 +2363,7 @@ class _HttpClientBasicCredentials // http://tools.ietf.org/html/draft-reschke-basicauth-enc-06. For // now always use UTF-8 encoding. String auth = - _CryptoUtils.bytesToBase64(_encodeString("$username:$password")); + _CryptoUtils.bytesToBase64(UTF8.encode("$username:$password")); return "Basic $auth"; } diff --git a/sdk/lib/io/io.dart b/sdk/lib/io/io.dart index 37a65c1226bf..e390d319ce4f 100644 --- a/sdk/lib/io/io.dart +++ b/sdk/lib/io/io.dart @@ -4,7 +4,7 @@ /** * File, socket, HTTP, and other I/O support for server applications. - * + * * The IO library is used for Dart server applications, * which run on a stand-alone Dart VM from the command line. * *This library does not work in browser based applications.* @@ -23,7 +23,6 @@ import 'dart:convert'; import 'dart:isolate'; import 'dart:json' as JSON; import 'dart:math'; -import 'dart:utf'; import 'dart:typed_data'; part 'bytes_builder.dart'; diff --git a/sdk/lib/io/io_sink.dart b/sdk/lib/io/io_sink.dart index b954a86d5ffe..ac4ce0a1daab 100644 --- a/sdk/lib/io/io_sink.dart +++ b/sdk/lib/io/io_sink.dart @@ -19,7 +19,7 @@ part of dart.io; */ abstract class IOSink implements StreamSink>, StringSink { factory IOSink(StreamConsumer> target, - {Encoding encoding: Encoding.UTF_8}) + {Encoding encoding: UTF8}) => new _IOSinkImpl(target, encoding); /** @@ -204,7 +204,7 @@ class _IOSinkImpl extends _StreamSinkImpl> implements IOSink { } } if (string.isEmpty) return; - add(_encodeString(string, _encoding)); + add(_encoding.encode(string)); } void writeAll(Iterable objects, [String separator = ""]) { diff --git a/sdk/lib/io/process.dart b/sdk/lib/io/process.dart index 0f89a300af56..5ca159765699 100644 --- a/sdk/lib/io/process.dart +++ b/sdk/lib/io/process.dart @@ -129,7 +129,7 @@ abstract class Process { * * The encoding used for decoding `stdout` and `stderr` into text is * controlled through [stdoutEncoding] and [stderrEncoding]. The - * default encoding is `Encoding.SYSTEM`. If `null` is used no + * default encoding is [SYSTEM_ENCODING]. If `null` is used no * decoding will happen and the [ProcessResult] will hold binary * data. * @@ -144,8 +144,8 @@ abstract class Process { Map environment, bool includeParentEnvironment: true, bool runInShell: false, - Encoding stdoutEncoding: Encoding.SYSTEM, - Encoding stderrEncoding: Encoding.SYSTEM}); + Encoding stdoutEncoding: SYSTEM_ENCODING, + Encoding stderrEncoding: SYSTEM_ENCODING}); /** @@ -164,8 +164,8 @@ abstract class Process { Map environment, bool includeParentEnvironment: true, bool runInShell: false, - Encoding stdoutEncoding: Encoding.SYSTEM, - Encoding stderrEncoding: Encoding.SYSTEM}); + Encoding stdoutEncoding: SYSTEM_ENCODING, + Encoding stderrEncoding: SYSTEM_ENCODING}); /** * Returns the standard output stream of the process as a [:Stream:]. diff --git a/sdk/lib/io/stdio.dart b/sdk/lib/io/stdio.dart index d69839e895f2..372552624bc0 100644 --- a/sdk/lib/io/stdio.dart +++ b/sdk/lib/io/stdio.dart @@ -30,10 +30,14 @@ class _StdStream extends Stream> { class _StdinEventSink implements EventSink { - Function add; - Function addError; - Function close; - _StdinEventSink(this.add, this.addError, this.close); + Function _add; + Function _addError; + Function _close; + _StdinEventSink(this._add, this._addError, this._close); + + void add(String string) => _add(string); + void addError(errorEvent) => _addError(errorEvent); + void close() => _close(); } /** @@ -54,42 +58,72 @@ class Stdin extends _StdStream implements Stream> { * If end-of-file is reached after some data has already been read, that data * is returned. */ - String readLineSync({Encoding encoding: Encoding.SYSTEM, + String readLineSync({Encoding encoding: SYSTEM_ENCODING, bool retainNewlines: false}) { - var decoder = new StringDecoder(encoding)._decoder; + const CR = 13; + const LF = 10; var line = new StringBuffer(); bool end = false; bool lastCharWasCR = false; - var sink = new _StdinEventSink( - (char) { - if (!retainNewlines && char == '\r') { - if (lastCharWasCR) line.write('\r'); - lastCharWasCR = true; - return; - } else if (char == '\n') { - end = true; - if (!retainNewlines) return; - } else if (lastCharWasCR) { - lastCharWasCR = false; - line.write('\r'); - } - line.write(char); - }, - (error) { - throw error; - }, () { - if (lastCharWasCR) line.write('\r'); - }); + var error; + + StreamController> controller = + new StreamController>(sync: true); + Stream stream = controller.stream.transform(encoding.decoder); + stream.listen((String str) { + line.write(str); + }, onError: (e) { + error = e; + }, onDone: () { + end = true; + }); bool empty = true; while (!end) { int b = readByteSync(); - if (b >= 0) { - empty = false; - decoder.handleData([b], sink); + + if (b < 0) { + // We didn't write the carriage return in case a line feed would be + // the next character. Add it now. + if (lastCharWasCR && !retainNewlines) controller.add([CR]); + controller.close(); } else { - decoder.handleDone(sink); - break; + empty = false; + // We consider \r\n and \n as new lines. + // A \r on its own is treated like a normal character. + + if (b == CR) { + if (lastCharWasCR && !retainNewlines) { + // We didn't write the carriage return in case a line feed would be + // the next character. + // Add it now (since we treat it like a normal character now). + controller.add([CR]); + } + // We add the carriage return only if we keep new lines. + // Otherwise we need to wait for the next character (in case it is + // a line feed). + if (retainNewlines) controller.add([b]); + lastCharWasCR = true; + } else if (b == LF) { + end = true; + // We don't care if there was a carriage return before. If we keep + // the line separators it has already been added to the controller. + // Otherwise we don't want it anyway. + if (retainNewlines) controller.add([b]); + controller.close(); + } else { + // Since the current character is not a line feed we flush the + // carriage return we didn't write last iteration. + if (lastCharWasCR) { + controller.add([CR]); + lastCharWasCR = false; + } + controller.add([b]); + } + } + if (error != null) { + // Error during decoding. + throw error; } } diff --git a/sdk/lib/io/string_transformer.dart b/sdk/lib/io/string_transformer.dart index 5324d6913fe8..91787573df72 100644 --- a/sdk/lib/io/string_transformer.dart +++ b/sdk/lib/io/string_transformer.dart @@ -4,357 +4,126 @@ part of dart.io; +const SYSTEM_ENCODING = const SystemEncoding(); + /** - * String encodings. + * The system encoding is the current code page on Windows and UTF-8 on + * Linux and Mac. */ -class Encoding { - static const Encoding UTF_8 = const Encoding._internal("utf-8"); - static const Encoding ISO_8859_1 = const Encoding._internal("iso-8859-1"); - static const Encoding ASCII = const Encoding._internal("us-ascii"); - - /** - * SYSTEM encoding is the current code page on Windows and UTF-8 on - * Linux and Mac. - */ - static const Encoding SYSTEM = const Encoding._internal("system"); - - // All aliasses (in lowercase) of supported encoding from - // http://www.iana.org/assignments/character-sets/character-sets.xml. - static Map _nameToEncoding = { - // ISO_8859-1:1987. - "iso_8859-1:1987": ISO_8859_1, - "iso-ir-100": ISO_8859_1, - "iso_8859-1": ISO_8859_1, - "iso-8859-1": ISO_8859_1, - "latin1": ISO_8859_1, - "l1": ISO_8859_1, - "ibm819": ISO_8859_1, - "cp819": ISO_8859_1, - "csisolatin1": ISO_8859_1, +class SystemEncoding extends Encoding { + const SystemEncoding(); - // US-ASCII. - "iso-ir-6": ASCII, - "ansi_x3.4-1968": ASCII, - "ansi_x3.4-1986": ASCII, - "iso_646.irv:1991": ASCII, - "iso646-us": ASCII, - "us-ascii": ASCII, - "us": ASCII, - "ibm367": ASCII, - "cp367": ASCII, - "csascii": ASCII, - "ascii": ASCII, // This is not in the IANA official names. + List encode(String input) => encoder.convert(input); + String decode(List encoded) => decoder.convert(encoded); - // UTF-8. - "csutf8": UTF_8, - "utf-8": UTF_8 - }; - - /** - * Gets an [Encoding] object from the name of the character set - * name. The names used are the IANA official names for the - * character set (see - * http://www.iana.org/assignments/character-sets/character-sets.xml). - * - * The [name] passed is case insensitive. - * - * If character set is not supported [:null:] is returned. - */ - static Encoding fromName(String name) { - if (name == null) return null; - name = name.toLowerCase(); - return _nameToEncoding[name]; + Converter> get encoder { + if (Platform.operatingSystem == "windows") { + return const _WindowsCodePageEncoder(); + } else { + return const Utf8Encoder(); + } } - /** - * Name of the encoding. This will be the lower-case version of one of the - * IANA official names for the character set (see - * http://www.iana.org/assignments/character-sets/character-sets.xml) - */ - final String name; - - const Encoding._internal(String this.name); + Converter, String> get decoder { + if (Platform.operatingSystem == "windows") { + return const _WindowsCodePageDecoder(); + } else { + return const Utf8Decoder(); + } + } } -const UTF_8 = Encoding.UTF_8; -const ISO_8859_1 = Encoding.ISO_8859_1; -const ASCII = Encoding.ASCII; +class _WindowsCodePageEncoder extends Converter> { -/** - * Stream transformer that can decode a stream of bytes into a stream of - * strings using [encoding]. - * - * Invalid or forbidden byte-sequences will not produce errors, but will instead - * insert [replacementChar] in the decoded strings. - */ -class StringDecoder implements StreamTransformer, String> { - var _decoder; - - /** - * Decodes a stream of bytes into a `String` with an optional - * [encoding] and [replacementChar]. - * - * The default value for [encoding] is [Encoding.UTF_8]. - * - * The default value for [replacementChar] is code point U+FFFD. - * - * Completes with the decoded `String` when the stream is done. - */ - static Future decode( - Stream> stream, - [Encoding encoding = Encoding.UTF_8, - int replacementChar = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { - return stream - .transform(new StringDecoder(encoding, replacementChar)) - .fold( - new StringBuffer(), - (prev, data) => prev..write(data)) - .then((sb) => sb.toString()); - } + const _WindowsCodePageEncoder(); - /** - * Create a new [StringDecoder] with an optional [encoding] and - * [replacementChar]. - * - * The default value for [encoding] is [Encoding.UTF_8]. - * - * The default value for [replacementChar] is code point U+FFFD. - */ - StringDecoder([Encoding encoding = Encoding.UTF_8, int replacementChar]) { - switch (encoding) { - case Encoding.UTF_8: - if (replacementChar == null) { - replacementChar = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT; - } - _decoder = new Utf8DecoderTransformer(replacementChar); - break; - case Encoding.ASCII: - if (replacementChar == null) { - replacementChar = '?'.codeUnitAt(0); - } else if (replacementChar > 127) { - throw new ArgumentError("Invalid replacement character for ASCII"); - } - _decoder = new _AsciiDecoder(replacementChar); - break; - case Encoding.ISO_8859_1: - if (replacementChar == null) { - replacementChar = '?'.codeUnitAt(0); - } else if (replacementChar > 255) { - throw new ArgumentError( - "Invalid replacement character for ISO_8859_1"); - } - _decoder = new _Latin1Decoder(replacementChar); - break; - case Encoding.SYSTEM: - if (Platform.operatingSystem == "windows") { - _decoder = new _WindowsCodePageDecoder(); - } else { - if (replacementChar != null) { - // TODO(ajohnsen): Handle replacement character. - throw new UnsupportedError( - "Replacement character is not supported for SYSTEM encoding"); - } - _decoder = new Utf8DecoderTransformer(); - } - break; - default: - throw new ArgumentError("Unsupported encoding '$encoding'"); + List convert(String input) { + List encoded = _encodeString(input); + if (encoded == null) { + throw new FormatException("Invalid character for encoding"); } + return encoded; } - Stream bind(Stream> stream) => _decoder.bind(stream); -} - - -/** - * Stream transformer that can encode a stream of strings info a stream of - * bytes using [encoding]. - * - * Strings that cannot be represented in the given encoding will result in an - * error and a close event on the stream. - */ -class StringEncoder implements StreamTransformer> { - var _encoder; - /** - * Create a new [StringDecoder] with an optional [encoding] and - * [replacementChar]. + * Starts a chunked conversion. */ - StringEncoder([Encoding encoding = Encoding.UTF_8]) { - switch (encoding) { - case Encoding.UTF_8: - _encoder = new Utf8EncoderTransformer(); - break; - case Encoding.ASCII: - _encoder = new _AsciiEncoder(); - break; - case Encoding.ISO_8859_1: - _encoder = new _Latin1Encoder(); - break; - case Encoding.SYSTEM: - if (Platform.operatingSystem == "windows") { - _encoder = new _WindowsCodePageEncoder(); - } else { - _encoder = new Utf8EncoderTransformer(); - } - break; - default: - throw new ArgumentError("Unsupported encoding '$encoding'"); - } + StringConversionSink startChunkedConversion( + ChunkedConversionSink> sink) { + return new _WindowsCodePageEncoderSink(sink); } - Stream> bind(Stream stream) => _encoder.bind(stream); -} - + // Override the base-class' bind, to provide a better type. + Stream> bind(Stream stream) => super.bind(stream); -// Utility function to synchronously decode a list of bytes. -String _decodeString(List bytes, [Encoding encoding = Encoding.UTF_8]) { - if (bytes.length == 0) return ""; - var string; - var error; - var controller = new StreamController(sync: true); - controller.stream - .transform(new StringDecoder(encoding)) - .listen((data) => string = data, - onError: (e) => error = e); - controller.add(bytes); - controller.close(); - if (error != null) throw error; - assert(string != null); - return string; -} - - -// Utility function to synchronously decode a utf8-encoded list of bytes, -// throwing on error. -String _decodeUtf8Strict(List bytes) { - if (bytes.length == 0) return ""; - var string; - var error; - var controller = new StreamController(sync: true); - controller.stream - .transform(new Utf8DecoderTransformer(null)) - .listen((data) => string = data, - onError: (e) => error = e); - controller.add(bytes); - controller.close(); - if (error != null) throw error; - assert(string != null); - return string; + external static List _encodeString(String string); } +class _WindowsCodePageEncoderSink extends StringConversionSinkBase { + // TODO(floitsch): provide more efficient conversions when the input is + // not a String. -// Utility function to synchronously encode a String. -// Will throw an exception if the encoding is invalid. -List _encodeString(String string, [Encoding encoding = Encoding.UTF_8]) { - if (string.length == 0) return []; - var bytes; - var controller = new StreamController(sync: true); - controller.stream - .transform(new StringEncoder(encoding)) - .listen((data) => bytes = data); - controller.add(string); - controller.close(); - assert(bytes != null); - return bytes; -} - + final ChunkedConversionSink> _sink; -abstract class _SingleByteDecoder - extends StreamEventTransformer, String> { - final int _replacementChar; + _WindowsCodePageEncoderSink(this._sink); - _SingleByteDecoder(this._replacementChar); + void close() { + _sink.close(); + } - void handleData(List data, EventSink sink) { - var buffer = new List(data.length); - for (int i = 0; i < data.length; i++) { - int char = _decodeByte(data[i]); - if (char < 0) char = _replacementChar; - buffer[i] = char; + void add(String string) { + List encoded = _WindowsCodePageEncoder._encodeString(string); + if (encoded == null) { + throw new FormatException("Invalid character for encoding"); } - sink.add(new String.fromCharCodes(buffer)); + _sink.add(encoded); } - int _decodeByte(int byte); -} - - -// Utility class for decoding ascii data delivered as a stream of -// bytes. -class _AsciiDecoder extends _SingleByteDecoder { - _AsciiDecoder(int replacementChar) : super(replacementChar); - - int _decodeByte(int byte) => ((byte & 0x7f) == byte) ? byte : -1; + void addSlice(String source, int start, int end, bool isLast) { + if (start != 0 || end != source.length) { + source = source.substring(start, end); + } + add(source); + if (isLast) close(); + } } -// Utility class for decoding Latin-1 data delivered as a stream of -// bytes. -class _Latin1Decoder extends _SingleByteDecoder { - _Latin1Decoder(int replacementChar) : super(replacementChar); - - int _decodeByte(int byte) => ((byte & 0xFF) == byte) ? byte : -1; -} +class _WindowsCodePageDecoder extends Converter, String> { + const _WindowsCodePageDecoder(); -abstract class _SingleByteEncoder - extends StreamEventTransformer> { - void handleData(String data, EventSink> sink) { - var bytes = _encode(data); - if (bytes == null) { - sink.addError(new FormatException("Invalid character for encoding")); - sink.close(); - } else { - sink.add(bytes); - } + String convert(List input) { + return _decodeBytes(input); } - List _encode(String string); -} - - -// Utility class for encoding a string into an ASCII byte stream. -class _AsciiEncoder extends _SingleByteEncoder { - List _encode(String string) { - var bytes = string.codeUnits; - for (var byte in bytes) { - if (byte > 127) return null; - } - return bytes; + /** + * Starts a chunked conversion. + */ + ByteConversionSink startChunkedConversion( + ChunkedConversionSink sink) { + return new _WindowsCodePageDecoderSink(sink); } -} + // Override the base-class' bind, to provide a better type. + Stream bind(Stream> stream) => super.bind(stream); -// Utility class for encoding a string into a Latin1 byte stream. -class _Latin1Encoder extends _SingleByteEncoder { - List _encode(String string) { - var bytes = string.codeUnits; - for (var byte in bytes) { - if (byte > 255) return null; - } - return bytes; - } + external static String _decodeBytes(List bytes); } +class _WindowsCodePageDecoderSink extends ByteConversionSinkBase { + // TODO(floitsch): provide more efficient conversions when the input is + // a slice. -// Utility class for encoding a string into a current windows -// code page byte list. -// Implemented on top of a _SingleByteEncoder, even though it's not really a -// single byte encoder, to avoid copying boilerplate. -class _WindowsCodePageEncoder extends _SingleByteEncoder { - List _encode(String string) => _encodeString(string); - - external static List _encodeString(String string); -} + final ChunkedConversionSink _sink; + _WindowsCodePageDecoderSink(this._sink); -// Utility class for decoding Windows current code page data delivered -// as a stream of bytes. -class _WindowsCodePageDecoder extends StreamEventTransformer, String> { - void handleData(List data, EventSink sink) { - sink.add(_decodeBytes(data)); + void close() { + _sink.close(); } - external static String _decodeBytes(List bytes); + void add(List bytes) { + _sink.add(_WindowsCodePageDecoder._decodeBytes(bytes)); + } } diff --git a/sdk/lib/io/websocket_impl.dart b/sdk/lib/io/websocket_impl.dart index aec29197075b..f85ddf288293 100644 --- a/sdk/lib/io/websocket_impl.dart +++ b/sdk/lib/io/websocket_impl.dart @@ -95,7 +95,7 @@ class _WebSocketProtocolTransformer extends StreamEventTransformer { _currentMessageType = _WebSocketMessageType.TEXT; _controller = new StreamController(sync: true); _controller.stream - .transform(new Utf8DecoderTransformer(null)) + .transform(UTF8.decoder) .fold(new StringBuffer(), (buffer, str) => buffer..write(str)) .then((buffer) { sink.add(buffer.toString()); @@ -304,7 +304,7 @@ class _WebSocketProtocolTransformer extends StreamEventTransformer { throw new WebSocketException("Protocol error"); } if (_controlPayload.length > 2) { - closeReason = _decodeUtf8Strict(_controlPayload.sublist(2)); + closeReason = UTF8.decode(_controlPayload.sublist(2)); } } _state = CLOSED; @@ -463,7 +463,7 @@ class _WebSocketOutgoingTransformer extends StreamEventTransformer { if (message != null) { if (message is String) { opcode = _WebSocketOpcode.TEXT; - data = _encodeString(message); + data = UTF8.encode(message); } else { if (message is !List) { throw new ArgumentError(message); @@ -486,7 +486,7 @@ class _WebSocketOutgoingTransformer extends StreamEventTransformer { data.add((code >> 8) & 0xFF); data.add(code & 0xFF); if (reason != null) { - data.addAll(_encodeString(reason)); + data.addAll(UTF8.encode(reason)); } } addFrame(_WebSocketOpcode.CLOSE, data, sink); diff --git a/tests/language/language.status b/tests/language/language.status index 70f61d019371..1d8e77843e88 100644 --- a/tests/language/language.status +++ b/tests/language/language.status @@ -258,3 +258,6 @@ null_test/03: Fail # Issue 12445. [ $compiler == dart2js ] null_test/none: Fail # Issue 12482 + +[ $compiler == none && $runtime == drt ] +vm/reflect_core_vm_test: Fail # Temporarily marked as fail. diff --git a/tests/language/optimized_isempty_test.dart b/tests/language/optimized_isempty_test.dart new file mode 100644 index 000000000000..d223c8b1f19c --- /dev/null +++ b/tests/language/optimized_isempty_test.dart @@ -0,0 +1,21 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +// VMOptions=--optimization-counter-threshold=10 --no-use-osr + +// Test optimization and polymorphic inlining of String.isEmpty. + +import "package:expect/expect.dart"; + +test(s) => s.isEmpty; + +main() { + var x = "abc"; + var y = [123, 12345, 765]; + Expect.equals(false, test(x)); + Expect.equals(false, test(y)); + for (var i=0; i<20; i++) test(x); + Expect.equals(false, test(x)); + Expect.equals(false, test(y)); +} + diff --git a/tests/language/reflect_core_vm_test.dart b/tests/language/reflect_core_vm_test.dart new file mode 100644 index 000000000000..d6e52f1805e5 --- /dev/null +++ b/tests/language/reflect_core_vm_test.dart @@ -0,0 +1,20 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// Test reflection of private functions in core classes. + +import "package:expect/expect.dart"; +import "dart:mirrors"; + +main() { + var s = "string"; + var im = reflect(s); + try { + im.invoke(const Symbol("_setAt"), [0, 65]); + Expect.isTrue(false); // Unreachable. + } catch (e) { + Expect.equals(true, e is NoSuchMethodError); + } +} + diff --git a/tests/lib/convert/encoding_test.dart b/tests/lib/convert/encoding_test.dart new file mode 100644 index 000000000000..cbdf72d1fc8b --- /dev/null +++ b/tests/lib/convert/encoding_test.dart @@ -0,0 +1,39 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "package:expect/expect.dart"; +import 'dart:async'; +import 'dart:convert'; +import 'unicode_tests.dart'; +import '../../async_helper.dart'; + +void runTest(List bytes, expected) { + var controller = new StreamController(); + asyncStart(); + UTF8.decodeStream(controller.stream).then((decoded) { + Expect.equals(expected, decoded); + asyncEnd(); + }); + int i = 0; + while (i < bytes.length) { + List nextChunk = []; + for (int j = 0; j < 3; j++) { + if (i < bytes.length) { + nextChunk.add(bytes[i]); + i++; + } + } + controller.add(nextChunk); + } + controller.close(); +} + + +main() { + for (var test in UNICODE_TESTS) { + var bytes = test[0]; + var expected = test[1]; + runTest(bytes, expected); + } +} diff --git a/tests/standalone/coverage_test.dart b/tests/standalone/coverage_test.dart index ecf92eaf7ea3..c9a5c5fc1528 100644 --- a/tests/standalone/coverage_test.dart +++ b/tests/standalone/coverage_test.dart @@ -65,11 +65,11 @@ void main() { coverageToolProcess = process; coverageToolProcess.stdin.close(); var stdoutStringStream = coverageToolProcess.stdout - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); var stderrStringStream = coverageToolProcess.stderr - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); // Wait for 3 future events: stdout and stderr streams of the coverage diff --git a/tests/standalone/debugger/debug_lib.dart b/tests/standalone/debugger/debug_lib.dart index 7be6854ee083..0377ae9711bb 100644 --- a/tests/standalone/debugger/debug_lib.dart +++ b/tests/standalone/debugger/debug_lib.dart @@ -393,7 +393,7 @@ class Debugger { Debugger(this.targetProcess, this.script) { var stdoutStringStream = targetProcess.stdout - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); stdoutStringStream.listen((line) { print("TARG: $line"); @@ -406,7 +406,7 @@ class Debugger { }); var stderrStringStream = targetProcess.stderr - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); stderrStringStream.listen((line) { print("TARG: $line"); @@ -542,7 +542,7 @@ class Debugger { Socket.connect("127.0.0.1", portNumber).then((s) { s.setOption(SocketOption.TCP_NODELAY, true); this.socket = s; - var stringStream = socket.transform(new StringDecoder()); + var stringStream = socket.transform(UTF8.decoder); stringStream.listen((str) { try { responses.append(str); diff --git a/tests/standalone/io/file_error_test.dart b/tests/standalone/io/file_error_test.dart index 27753d6d4609..6ebb5b9ed760 100644 --- a/tests/standalone/io/file_error_test.dart +++ b/tests/standalone/io/file_error_test.dart @@ -5,6 +5,7 @@ // Dart test program for testing error handling in file I/O. import "package:expect/expect.dart"; +import "dart:convert"; import "dart:io"; import "dart:isolate"; @@ -205,7 +206,7 @@ void testReadAsTextNonExistent() { Expect.throws(() => file.readAsStringSync(), (e) => checkOpenNonExistentFileException(e)); - var readAsStringFuture = file.readAsString(encoding: Encoding.ASCII); + var readAsStringFuture = file.readAsString(encoding: ASCII); readAsStringFuture.then((data) => Expect.fail("Unreachable code")) .catchError((error) { checkOpenNonExistentFileException(error); @@ -226,7 +227,7 @@ testReadAsLinesNonExistent() { Expect.throws(() => file.readAsLinesSync(), (e) => checkOpenNonExistentFileException(e)); - var readAsLinesFuture = file.readAsLines(encoding: Encoding.ASCII); + var readAsLinesFuture = file.readAsLines(encoding: ASCII); readAsLinesFuture.then((data) => Expect.fail("Unreachable code")) .catchError((error) { checkOpenNonExistentFileException(error); diff --git a/tests/standalone/io/file_input_stream_test.dart b/tests/standalone/io/file_input_stream_test.dart index 021e19e68215..824cfce9f334 100644 --- a/tests/standalone/io/file_input_stream_test.dart +++ b/tests/standalone/io/file_input_stream_test.dart @@ -20,7 +20,7 @@ void testStringLineSplitter() { File file = new File(fileName); int linesRead = 0; var lineStream = file.openRead() - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); lineStream.listen((line) { linesRead++; @@ -232,7 +232,7 @@ void testStringLineSplitterEnding(String name, int length) { File file = new File(fileName); Expect.equals(length, file.openSync().lengthSync()); var lineStream = file.openRead() - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); int lineCount = 0; lineStream.listen( diff --git a/tests/standalone/io/file_test.dart b/tests/standalone/io/file_test.dart index 6a16efcfff93..28001ededbd8 100644 --- a/tests/standalone/io/file_test.dart +++ b/tests/standalone/io/file_test.dart @@ -7,6 +7,7 @@ import "package:expect/expect.dart"; import "package:path/path.dart"; import 'dart:async'; +import 'dart:convert'; import 'dart:collection'; import 'dart:io'; import 'dart:isolate'; @@ -393,13 +394,13 @@ class FileTest { List buffer = content.codeUnits; var output = file.openWrite(); output.write("abcdABCD"); - output.encoding = UTF_8; + output.encoding = UTF8; output.write("abcdABCD"); - output.encoding = ISO_8859_1; + output.encoding = LATIN1; output.write("abcdABCD"); output.encoding = ASCII; output.write("abcdABCD"); - output.encoding = UTF_8; + output.encoding = UTF8; output.write("æøå"); output.close(); output.done.then((_) { @@ -985,16 +986,16 @@ class FileTest { }); var name = getFilename("tests/vm/data/fixed_length_file"); var f = new File(name); - f.readAsString(encoding: UTF_8).then((text) { + f.readAsString(encoding: UTF8).then((text) { Expect.isTrue(text.endsWith("42 bytes.")); Expect.equals(42, text.length); var name = getDataFilename("tests/standalone/io/read_as_text.dat"); var f = new File(name); - f.readAsString(encoding: UTF_8).then((text) { + f.readAsString(encoding: UTF8).then((text) { Expect.equals(6, text.length); var expected = [955, 120, 46, 32, 120, 10]; Expect.listEquals(expected, text.codeUnits); - f.readAsString(encoding: ISO_8859_1).then((text) { + f.readAsString(encoding: LATIN1).then((text) { Expect.equals(7, text.length); var expected = [206, 187, 120, 46, 32, 120, 10]; Expect.listEquals(expected, text.codeUnits); @@ -1017,7 +1018,7 @@ class FileTest { }); var name = getFilename("tests/vm/data/empty_file"); var f = new File(name); - f.readAsString(encoding: UTF_8).then((text) { + f.readAsString(encoding: UTF8).then((text) { port.toSendPort().send(text.length); return true; }); @@ -1033,11 +1034,18 @@ class FileTest { Expect.equals(6, text.length); var expected = [955, 120, 46, 32, 120, 10]; Expect.listEquals(expected, text.codeUnits); - text = new File(name).readAsStringSync(encoding: ASCII); - // Default replacement character is '?', char code 63. - expected = [63, 63, 120, 46, 32, 120, 10]; + // First character is not ASCII. The default ASCII decoder will throw. + Expect.throws(() => new File(name).readAsStringSync(encoding: ASCII), + (e) => e is FormatException); + // We can use an ASCII decoder that inserts the replacement character. + var lenientAscii = const AsciiCodec(allowInvalid: true); + text = new File(name).readAsStringSync(encoding: lenientAscii); + // Default replacement character is the Unicode replacement character. + expected = [UNICODE_REPLACEMENT_CHARACTER_RUNE, + UNICODE_REPLACEMENT_CHARACTER_RUNE, + 120, 46, 32, 120, 10]; Expect.listEquals(expected, text.codeUnits); - text = new File(name).readAsStringSync(encoding: ISO_8859_1); + text = new File(name).readAsStringSync(encoding: LATIN1); expected = [206, 187, 120, 46, 32, 120, 10]; Expect.equals(7, text.length); Expect.listEquals(expected, text.codeUnits); @@ -1057,7 +1065,7 @@ class FileTest { }); var name = getFilename("tests/vm/data/fixed_length_file"); var f = new File(name); - f.readAsLines(encoding: UTF_8).then((lines) { + f.readAsLines(encoding: UTF8).then((lines) { Expect.equals(1, lines.length); var line = lines[0]; Expect.isTrue(line.endsWith("42 bytes.")); @@ -1091,10 +1099,10 @@ class FileTest { var readAsBytesFuture = f.readAsBytes(); readAsBytesFuture.then((bytes) => Expect.fail("no bytes expected")) .catchError((e) { - var readAsStringFuture = f.readAsString(encoding: UTF_8); + var readAsStringFuture = f.readAsString(encoding: UTF8); readAsStringFuture.then((text) => Expect.fail("no text expected")) .catchError((e) { - var readAsLinesFuture = f.readAsLines(encoding: UTF_8); + var readAsLinesFuture = f.readAsLines(encoding: UTF8); readAsLinesFuture.then((lines) => Expect.fail("no lines expected")) .catchError((e) { port.toSendPort().send(1); diff --git a/tests/standalone/io/http_auth_digest_test.dart b/tests/standalone/io/http_auth_digest_test.dart index 2a3828d356b5..5512cd02d2f8 100644 --- a/tests/standalone/io/http_auth_digest_test.dart +++ b/tests/standalone/io/http_auth_digest_test.dart @@ -7,7 +7,6 @@ import "package:expect/expect.dart"; import 'dart:async'; import 'dart:io'; import 'dart:isolate'; -import 'dart:utf'; class Server { HttpServer server; diff --git a/tests/standalone/io/http_auth_test.dart b/tests/standalone/io/http_auth_test.dart index 9869a94bcec2..d259bdf03000 100644 --- a/tests/standalone/io/http_auth_test.dart +++ b/tests/standalone/io/http_auth_test.dart @@ -7,7 +7,7 @@ import "package:expect/expect.dart"; import 'dart:async'; import 'dart:io'; import 'dart:isolate'; -import 'dart:utf'; +import 'dart:convert'; class Server { HttpServer server; @@ -42,7 +42,7 @@ class Server { List tokens = authorization.split(" "); Expect.equals("Basic", tokens[0]); String auth = - CryptoUtils.bytesToBase64(encodeUtf8("$username:$password")); + CryptoUtils.bytesToBase64(UTF8.encode("$username:$password")); if (passwordChanged && auth != tokens[1]) { response.statusCode = HttpStatus.UNAUTHORIZED; response.headers.set(HttpHeaders.WWW_AUTHENTICATE, diff --git a/tests/standalone/io/http_proxy_test.dart b/tests/standalone/io/http_proxy_test.dart index 3b56004e08fe..81806ab65c2c 100644 --- a/tests/standalone/io/http_proxy_test.dart +++ b/tests/standalone/io/http_proxy_test.dart @@ -7,7 +7,7 @@ import "package:expect/expect.dart"; import "package:path/path.dart"; import "dart:async"; import "dart:io"; -import 'dart:utf'; +import 'dart:convert'; class Server { HttpServer server; @@ -163,7 +163,7 @@ class ProxyServer { List tokens = authorization.split(" "); Expect.equals("Basic", tokens[0]); String auth = - CryptoUtils.bytesToBase64(encodeUtf8("$username:$password")); + CryptoUtils.bytesToBase64(UTF8.encode("$username:$password")); if (auth != tokens[1]) { basicAuthenticationRequired(request); return; diff --git a/tests/standalone/io/process_run_output_test.dart b/tests/standalone/io/process_run_output_test.dart index efa98927a5cd..28d9fa980224 100644 --- a/tests/standalone/io/process_run_output_test.dart +++ b/tests/standalone/io/process_run_output_test.dart @@ -6,10 +6,11 @@ // non-interactive processes started with Process.run. import "package:expect/expect.dart"; +import "dart:convert"; import "dart:io"; import "process_test_util.dart"; -checkOutput(encoding, output) { +checkOutput(String encoding, output) { if (encoding == 'ascii') { Expect.equals(output, 'abc'); } else if (encoding == 'latin1') { @@ -22,14 +23,14 @@ checkOutput(encoding, output) { } } -test(scriptFile, encoding, stream) { +test(scriptFile, String encoding, stream) { var enc; if (encoding == 'ascii') { - enc = Encoding.ASCII; + enc = ASCII; } else if (encoding == 'latin1') { - enc = Encoding.ISO_8859_1; + enc = LATIN1; } else if (encoding == 'utf8') { - enc = Encoding.UTF_8; + enc = UTF8; } else if (encoding == 'binary') { enc = null; } diff --git a/tests/standalone/io/process_std_io_script2.dart b/tests/standalone/io/process_std_io_script2.dart index db5792956cd0..0bd1a3e580ed 100644 --- a/tests/standalone/io/process_std_io_script2.dart +++ b/tests/standalone/io/process_std_io_script2.dart @@ -5,6 +5,7 @@ // Utility script to echo strings in various formats to stdout or // stderr. +import "dart:convert"; import "dart:io"; writeData(data, encoding, stream) { @@ -34,11 +35,11 @@ main() { if (options.arguments.length > 1) { var stream = options.arguments[1]; if (options.arguments[0] == "ascii") { - writeData(asciiString, Encoding.ASCII, stream); + writeData(asciiString, ASCII, stream); } else if (options.arguments[0] == "latin1") { - writeData(latin1String, Encoding.ISO_8859_1, stream); + writeData(latin1String, LATIN1, stream); } else if (options.arguments[0] == "utf8") { - writeData(utf8String, Encoding.UTF_8, stream); + writeData(utf8String, UTF8, stream); } else if (options.arguments[0] == "binary") { writeData(binary, null, stream); } diff --git a/tests/standalone/io/process_stdin_transform_unsubscribe_script.dart b/tests/standalone/io/process_stdin_transform_unsubscribe_script.dart index 06b42d8c610d..910af044a9d3 100644 --- a/tests/standalone/io/process_stdin_transform_unsubscribe_script.dart +++ b/tests/standalone/io/process_stdin_transform_unsubscribe_script.dart @@ -10,7 +10,7 @@ import "dart:io"; main() { var subscription; subscription = stdin - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()) .listen((String line) { // Unsubscribe after the first line. diff --git a/tests/standalone/io/regress_10026_test.dart b/tests/standalone/io/regress_10026_test.dart index 6c386c7cad88..1b7644c8aadb 100644 --- a/tests/standalone/io/regress_10026_test.dart +++ b/tests/standalone/io/regress_10026_test.dart @@ -4,6 +4,7 @@ import "package:expect/expect.dart"; import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; @@ -13,7 +14,7 @@ void testZLibInflate_regress10026() { var controller = new StreamController(sync: true); controller.stream .transform(ZLIB.decoder) - .transform(new StringDecoder()) + .transform(UTF8.decoder) .fold(new StringBuffer(), (buffer, s) { buffer.write(s); return buffer; diff --git a/tests/standalone/io/secure_socket_renegotiate_client.dart b/tests/standalone/io/secure_socket_renegotiate_client.dart index 49d9c99e9a64..0c724120fd97 100644 --- a/tests/standalone/io/secure_socket_renegotiate_client.dart +++ b/tests/standalone/io/secure_socket_renegotiate_client.dart @@ -44,7 +44,7 @@ void runClient(int port) { expectEquals('CN=localhost', certificate.subject); expectEquals('CN=myauthority', certificate.issuer); StreamIterator input = new StreamIterator(socket - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter())); socket.writeln('first'); input.moveNext() diff --git a/tests/standalone/io/secure_socket_renegotiate_test.dart b/tests/standalone/io/secure_socket_renegotiate_test.dart index 54736a8ea09d..ee115a91ae7b 100644 --- a/tests/standalone/io/secure_socket_renegotiate_test.dart +++ b/tests/standalone/io/secure_socket_renegotiate_test.dart @@ -30,7 +30,7 @@ Future runServer() { Expect.isNull(socket.peerCertificate); StreamIterator input = - new StreamIterator(socket.transform(new StringDecoder()) + new StreamIterator(socket.transform(UTF8.decoder) .transform(new LineSplitter())); input.moveNext().then((success) { Expect.isTrue(success); diff --git a/tests/standalone/io/stdin_sync_script.dart b/tests/standalone/io/stdin_sync_script.dart index d2bc4ad4c2cf..ed60a9c4fcda 100644 --- a/tests/standalone/io/stdin_sync_script.dart +++ b/tests/standalone/io/stdin_sync_script.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import "dart:convert"; import "dart:io"; import "dart:json"; @@ -9,7 +10,7 @@ void main() { var arguments = new Options().arguments; int i = 0; String line; - while ((line = stdin.readLineSync(encoding: Encoding.UTF_8)) != null) { + while ((line = stdin.readLineSync(encoding: UTF8)) != null) { if (parse(arguments[i]) != line) throw "bad line at $i: ${line.codeUnits}"; i++; } diff --git a/tests/standalone/io/stdin_sync_test.dart b/tests/standalone/io/stdin_sync_test.dart index 0ac4e9df5d50..c0477c01ffe7 100644 --- a/tests/standalone/io/stdin_sync_test.dart +++ b/tests/standalone/io/stdin_sync_test.dart @@ -17,14 +17,14 @@ void testReadByte() { process.stdin.write(line); process.stdin.close(); process.stderr - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()) .fold(new StringBuffer(), (b, d) => b..write(d)) .then((data) { if (data.toString() != '') throw "Bad output: '$data'"; }); process.stdout - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()) .fold(new StringBuffer(), (b, d) => b..write(d)) .then((data) { diff --git a/tests/standalone/io/stdout_stderr_test.dart b/tests/standalone/io/stdout_stderr_test.dart index 1f7ac91bf276..296a8d6415a7 100644 --- a/tests/standalone/io/stdout_stderr_test.dart +++ b/tests/standalone/io/stdout_stderr_test.dart @@ -4,12 +4,13 @@ import "package:expect/expect.dart"; import "dart:async"; +import "dart:convert"; import "dart:io"; callIOSink(IOSink sink) { // Call all methods on IOSink. - sink.encoding = Encoding.ASCII; - Expect.equals(Encoding.ASCII, sink.encoding); + sink.encoding = ASCII; + Expect.equals(ASCII, sink.encoding); sink.write("Hello\n"); sink.writeln("Hello"); sink.writeAll(["H", "e", "l", "lo\n"]); diff --git a/tests/standalone/io/string_decoder_test.dart b/tests/standalone/io/string_decoder_test.dart deleted file mode 100644 index 6e96000093dd..000000000000 --- a/tests/standalone/io/string_decoder_test.dart +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import "package:expect/expect.dart"; -import "dart:async"; -import "dart:io"; - -void testTransform() { - // Code point U+10FFFF is the largest code point supported by Dart. - var controller = new StreamController(sync: true); - controller.add([0xf0, 0x90, 0x80, 0x80]); // U+10000 - controller.add([0xf4, 0x8f, 0xbf, 0xbf]); // U+10FFFF - controller.add([0xf4, 0x90, 0x80, 0x80]); // U+110000 - controller.add([0xfa, 0x80, 0x80, 0x80, 0x80]); // U+2000000 - controller.add([0xfd, 0x80, 0x80, 0x80, 0x80, 0x80]); // U+40000000 - controller.close(); - - var decoder = new StringDecoder(Encoding.UTF_8, '?'.codeUnitAt(0)); - var stream = controller.stream.transform(decoder); - stream.fold( - new StringBuffer(), - (b, e) { - b.write(e); - return b; - }) - .then((b) => b.toString()) - .then((decoded) { - Expect.equals(8, decoded.length); - - var replacementChar = '?'.codeUnitAt(0); - Expect.equals(0xd800, decoded.codeUnitAt(0)); - Expect.equals(0xdc00, decoded.codeUnitAt(1)); - Expect.equals(0xdbff, decoded.codeUnitAt(2)); - Expect.equals(0xdfff, decoded.codeUnitAt(3)); - Expect.equals(replacementChar, decoded.codeUnitAt(4)); - Expect.equals(replacementChar, decoded.codeUnitAt(5)); - Expect.equals(replacementChar, decoded.codeUnitAt(6)); - Expect.equals(replacementChar, decoded.codeUnitAt(7)); - }); -} - -void testDecode() { - // Code point U+10FFFF is the largest code point supported by Dart. - var controller = new StreamController(sync: true); - controller.add([0xf0, 0x90, 0x80, 0x80]); // U+10000 - controller.add([0xf4, 0x8f, 0xbf, 0xbf]); // U+10FFFF - controller.add([0xf4, 0x90, 0x80, 0x80]); // U+110000 - controller.add([0xfa, 0x80, 0x80, 0x80, 0x80]); // U+2000000 - controller.add([0xfd, 0x80, 0x80, 0x80, 0x80, 0x80]); // U+40000000 - controller.close(); - - StringDecoder.decode(controller.stream, - Encoding.UTF_8, - '?'.codeUnitAt(0)).then((decoded) { - Expect.equals(8, decoded.length); - - var replacementChar = '?'.codeUnitAt(0); - Expect.equals(0xd800, decoded.codeUnitAt(0)); - Expect.equals(0xdc00, decoded.codeUnitAt(1)); - Expect.equals(0xdbff, decoded.codeUnitAt(2)); - Expect.equals(0xdfff, decoded.codeUnitAt(3)); - Expect.equals(replacementChar, decoded.codeUnitAt(4)); - Expect.equals(replacementChar, decoded.codeUnitAt(5)); - Expect.equals(replacementChar, decoded.codeUnitAt(6)); - Expect.equals(replacementChar, decoded.codeUnitAt(7)); - }); -} - -void testInvalid() { - void invalid(var bytes, var outputLength) { - var controller = new StreamController(sync: true); - controller.add(bytes); - controller.close(); - controller.stream.transform(new StringDecoder()).listen((string) { - Expect.equals(outputLength, string.length); - for (var i = 0; i < outputLength; i++) { - Expect.equals(0xFFFD, string.codeUnitAt(i)); - } - }); - } - - invalid([0x80], 1); - invalid([0xff], 1); - invalid([0xf0, 0xc0], 1); - invalid([0xc0, 0x80], 1); - invalid([0xfd, 0x80, 0x80], 3); // Unfinished encoding. -} - -void main() { - testTransform(); - testDecode(); - testInvalid(); -} diff --git a/tests/standalone/io/string_transformer_test.dart b/tests/standalone/io/string_transformer_test.dart deleted file mode 100644 index eb6d16847695..000000000000 --- a/tests/standalone/io/string_transformer_test.dart +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import "dart:async"; -import "dart:convert"; -import "dart:io"; -import "dart:isolate"; -import "dart:utf"; - -import "package:expect/expect.dart"; - -void main() { - testUtf8(); - testLatin1(); - testAscii(); - testReadLine1(); - testReadLine2(); - testErrorHandler(); - testLatin1EncoderError(); -} - -void testUtf8() { - List data = [0x01, - 0x7f, - 0xc2, 0x80, - 0xdf, 0xbf, - 0xe0, 0xa0, 0x80, - 0xef, 0xbf, 0xbf, - 0xf0, 0x9d, 0x84, 0x9e, - 0x100, -0x1, -0xFF]; - var controller = new StreamController(sync: true); - controller.add(data); - controller.close(); - var stringStream = controller.stream - .transform(new StringDecoder(Encoding.UTF_8)); - stringStream.listen( - (s) { - Expect.equals(11, s.length); - Expect.equals(new String.fromCharCodes([0x01]), s[0]); - Expect.equals(new String.fromCharCodes([0x7f]), s[1]); - Expect.equals(new String.fromCharCodes([0x80]), s[2]); - Expect.equals(new String.fromCharCodes([0x7ff]), s[3]); - Expect.equals(new String.fromCharCodes([0x800]), s[4]); - Expect.equals(new String.fromCharCodes([0xffff]), s[5]); - Expect.equals(new String.fromCharCodes([0xffff]), s[5]); - - // Surrogate pair for U+1D11E. - Expect.equals(new String.fromCharCodes([0xd834, 0xdd1e]), - s.substring(6, 8)); - - Expect.equals(new String.fromCharCodes( - [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, - UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, - UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]), - s.substring(8, 11)); - }); -} - -void testLatin1() { - List data = [0x01, - 0x7f, - 0x44, 0x61, 0x72, 0x74, - 0x80, - 0xff, - 0x100, -0x1, -0xff]; - var controller = new StreamController(sync: true); - controller.add(data); - controller.close(); - var stream = controller.stream - .transform(new StringDecoder(Encoding.ISO_8859_1)); - stream.listen((s) { - Expect.equals(11, s.length); - Expect.equals(new String.fromCharCodes([0x01]), s[0]); - Expect.equals(new String.fromCharCodes([0x7f]), s[1]); - Expect.equals("Dart", s.substring(2, 6)); - Expect.equals(new String.fromCharCodes([0x80]), s[6]); - Expect.equals(new String.fromCharCodes([0xff]), s[7]); - Expect.equals('???', s.substring(8, 11)); - }); -} - -void testAscii() { - List data = [0x01, - 0x44, 0x61, 0x72, 0x74, - 0x7f, - 0xf4, 0x100, -0x1, -0xff]; - var controller = new StreamController(sync: true); - controller.add(data); - controller.close(); - var stream = controller.stream - .transform(new StringDecoder(Encoding.ASCII)); - stream.listen((s) { - Expect.equals(10, s.length); - Expect.equals(new String.fromCharCodes([0x01]), s[0]); - Expect.equals("Dart", s.substring(1, 5)); - Expect.equals(new String.fromCharCodes([0x7f]), s[5]); - Expect.equals('????', s.substring(6, 10)); - }); -} - -void testReadLine1() { - var controller = new StreamController(sync: true); - var stream = controller.stream - .transform(new StringDecoder()) - .transform(new LineSplitter()); - - var stage = 0; - - void stringData(line) { - Expect.equals(stage, 0); - Expect.equals("Line", line); - stage++; - } - - void streamClosed() { - Expect.equals(1, stage); - } - - stream.listen( - stringData, - onDone: streamClosed); - - // Note: codeUnits is fine. Text is ASCII. - controller.add("Line".codeUnits); - controller.close(); - Expect.equals(1, stage); -} - -void testReadLine2() { - var controller = new StreamController(sync: true); - - var stream = controller.stream - .transform(new StringDecoder()) - .transform(new LineSplitter()); - - var expectedLines = ['Line1', 'Line2','Line3', 'Line4', - '', '', '', '', '', '', - 'Line5', 'Line6']; - - var index = 0; - - stream.listen((line) { - Expect.equals(expectedLines[index++], line); - }); - - // Note: codeUnits is fine. Text is ASCII. - controller.add("Line1\nLine2\r\nLine3\rLi".codeUnits); - controller.add("ne4\n".codeUnits); - controller.add("\n\n\r\n\r\n\r\r".codeUnits); - controller.add("Line5\r".codeUnits); - controller.add("\nLine6\n".codeUnits); - controller.close(); - Expect.equals(expectedLines.length, index); -} - -class TestException implements Exception { - TestException(); -} - -void testErrorHandler() { - var controller = new StreamController(sync: true); - var errors = 0; - var stream = controller.stream - .transform(new StringDecoder()) - .transform(new LineSplitter()); - stream.listen( - (_) {}, - onDone: () { - Expect.equals(1, errors); - }, - onError: (error) { - errors++; - Expect.isTrue(error is TestException); - }); - controller.addError(new TestException()); - controller.close(); -} - -void testLatin1EncoderError() { - List data = [0x01, - 0x7f, - 0x44, 0x61, 0x72, 0x74, - 0x80, - 0xff, - 0x100]; - var controller = new StreamController(sync: true); - controller.add(new String.fromCharCodes(data)); - controller.close(); - var stream = controller.stream - .transform(new StringEncoder(Encoding.ISO_8859_1)); - stream.listen( - (s) { - Expect.fail("data not expected"); - }, - onError: (error) { - Expect.isTrue(error is FormatException); - }); - -} diff --git a/tests/standalone/io/web_socket_protocol_processor_test.dart b/tests/standalone/io/web_socket_protocol_processor_test.dart index 4d90d73f32c9..4354ab0d27d0 100644 --- a/tests/standalone/io/web_socket_protocol_processor_test.dart +++ b/tests/standalone/io/web_socket_protocol_processor_test.dart @@ -3,7 +3,7 @@ // BSD-style license that can be found in the LICENSE file. import "package:expect/expect.dart"; -import "dart:utf"; +import "dart:convert"; import "dart:math"; import "dart:async"; import "dart:collection"; @@ -41,7 +41,7 @@ class WebSocketMessageCollector { void onMessageData(buffer) { if (buffer is String) { - buffer = _encodeString(buffer); + buffer = UTF8.encode(buffer); } Expect.listEquals(expectedMessage, buffer); messageCount++; diff --git a/tests/standalone/vmservice/test_helper.dart b/tests/standalone/vmservice/test_helper.dart index d199d05b8d53..f70e0f0dc703 100644 --- a/tests/standalone/vmservice/test_helper.dart +++ b/tests/standalone/vmservice/test_helper.dart @@ -97,7 +97,7 @@ class TestLauncher { var portNumber; var blank; var first = true; - process.stdout.transform(new StringDecoder()) + process.stdout.transform(UTF8.decoder) .transform(new LineSplitter()).listen((line) { if (line.startsWith('VmService listening on port ')) { RegExp portExp = new RegExp(r"\d+"); @@ -116,7 +116,7 @@ class TestLauncher { } print(line); }); - process.stderr.transform(new StringDecoder()) + process.stderr.transform(UTF8.decoder) .transform(new LineSplitter()).listen((line) { print(line); }); diff --git a/tools/VERSION b/tools/VERSION index 5018971353d9..d90d5d2c2e26 100644 --- a/tools/VERSION +++ b/tools/VERSION @@ -1,4 +1,4 @@ MAJOR 0 MINOR 6 BUILD 21 -PATCH 2 +PATCH 3 diff --git a/tools/coverage.dart b/tools/coverage.dart index 350019de3c50..37bb2bf3af26 100644 --- a/tools/coverage.dart +++ b/tools/coverage.dart @@ -214,7 +214,7 @@ class Debugger { Debugger(this.targetProcess) { var stdoutStringStream = targetProcess.stdout - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); stdoutStringStream.listen((line) { if (showDebuggeeOutput) { @@ -231,7 +231,7 @@ class Debugger { }); var stderrStringStream = targetProcess.stderr - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineSplitter()); stderrStringStream.listen((line) { if (showDebuggeeOutput) { @@ -372,7 +372,7 @@ class Debugger { Socket.connect("127.0.0.1", portNumber).then((s) { socket = s; socket.setOption(SocketOption.TCP_NODELAY, true); - var stringStream = socket.transform(new StringDecoder()); + var stringStream = socket.transform(UTF8.decoder); stringStream.listen( (str) { try { diff --git a/tools/ddbg.dart b/tools/ddbg.dart index 18673a6dcb95..132be1046bba 100644 --- a/tools/ddbg.dart +++ b/tools/ddbg.dart @@ -590,7 +590,7 @@ void debuggerMain() { Socket.connect("127.0.0.1", 5858).then((s) { vmSock = s; vmSock.setOption(SocketOption.TCP_NODELAY, true); - var stringStream = vmSock.transform(new StringDecoder()); + var stringStream = vmSock.transform(UTF8.decoder); vmSubscription = stringStream.listen( (String data) { processVmData(data); @@ -604,7 +604,7 @@ void debuggerMain() { // TODO(floitsch): do we want to print the stack trace? quitShell(); }); - stdinSubscription = stdin.transform(new StringDecoder()) + stdinSubscription = stdin.transform(UTF8.decoder) .transform(new LineTransformer()) .listen((String line) => processCommand(line)); }); diff --git a/utils/testrunner/layout_test_controller.dart b/utils/testrunner/layout_test_controller.dart index 27cd1cbeadbc..31a35cf702c9 100644 --- a/utils/testrunner/layout_test_controller.dart +++ b/utils/testrunner/layout_test_controller.dart @@ -168,7 +168,7 @@ Future _processHelper(String command, List args, void _pipeStream(Stream stream, List destination, Function outputMonitor) { stream - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineTransformer()) .listen((String line) { if (outputMonitor != null) { diff --git a/utils/testrunner/pipeline_utils.dart b/utils/testrunner/pipeline_utils.dart index 72c0164fec5c..38ed392286fa 100644 --- a/utils/testrunner/pipeline_utils.dart +++ b/utils/testrunner/pipeline_utils.dart @@ -101,7 +101,7 @@ Future _processHelper(String command, List args, Future _pipeStream(Stream stream, List destination, Function outputMonitor) { return stream - .transform(new StringDecoder()) + .transform(UTF8.decoder) .transform(new LineTransformer()) .listen((String line) { if (config["immediate"] && line.startsWith('###')) { diff --git a/utils/tests/testrunner/http_client_tests/http_client_test.dart b/utils/tests/testrunner/http_client_tests/http_client_test.dart index 0016ce4df0b2..452c1ab57b39 100644 --- a/utils/tests/testrunner/http_client_tests/http_client_test.dart +++ b/utils/tests/testrunner/http_client_tests/http_client_test.dart @@ -21,7 +21,7 @@ main() { // Process the response. expect(response.statusCode, code); var sb = new StringBuffer(); - response.transform(new StringDecoder()) + response.transform(UTF8.decoder) .listen((data) { sb.write(data); }, onDone: () {