Skip to content

Commit

Permalink
Implement lexer int128 support (#11571)
Browse files Browse the repository at this point in the history
  • Loading branch information
@BlobCodes
BlobCodes authored Dec 18, 2021
1 parent fdcbcf4 commit bfddd26
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 33 deletions.
55 changes: 39 additions & 16 deletions spec/compiler/lexer/lexer_spec.cr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,16 +302,33 @@ describe "Lexer" do
assert_syntax_error "18446744073709551616_i32", "18446744073709551616 doesn't fit in an Int32"
assert_syntax_error "9999999999999999999_i32", "9999999999999999999 doesn't fit in an Int32"

assert_syntax_error "-9999999999999999999", "-9999999999999999999 doesn't fit in an Int64"
assert_syntax_error "-99999999999999999999", "-99999999999999999999 doesn't fit in an Int64"
assert_syntax_error "-11111111111111111111", "-11111111111111111111 doesn't fit in an Int64"
assert_syntax_error "-9223372036854775809", "-9223372036854775809 doesn't fit in an Int64"
assert_syntax_error "18446744073709551616", "18446744073709551616 doesn't fit in an UInt64"

assert_syntax_error "9223372036854775808_i128", "9223372036854775808 doesn't fit in an Int64. Int128 literals that don't fit in an Int64 are currently not supported"
assert_syntax_error "-9223372036854775809_i128", "-9223372036854775809 doesn't fit in an Int64. Int128 literals that don't fit in an Int64 are currently not supported"
assert_syntax_error "118446744073709551616_u128", "118446744073709551616 doesn't fit in an UInt64. UInt128 literals that don't fit in an UInt64 are currently not supported"
assert_syntax_error "18446744073709551616_u128", "18446744073709551616 doesn't fit in an UInt64. UInt128 literals that don't fit in an UInt64 are currently not supported"
assert_syntax_error "-9999999999999999999", "-9999999999999999999 doesn't fit in an Int64, try using the suffix i128"
assert_syntax_error "-99999999999999999999", "-99999999999999999999 doesn't fit in an Int64, try using the suffix i128"
assert_syntax_error "-11111111111111111111", "-11111111111111111111 doesn't fit in an Int64, try using the suffix i128"
assert_syntax_error "-9223372036854775809", "-9223372036854775809 doesn't fit in an Int64, try using the suffix i128"
assert_syntax_error "118446744073709551616", "118446744073709551616 doesn't fit in an UInt64, try using the suffix i128"
assert_syntax_error "18446744073709551616", "18446744073709551616 doesn't fit in an UInt64, try using the suffix i128"

assert_syntax_error "340282366920938463463374607431768211456", "340282366920938463463374607431768211456 doesn't fit in an UInt64"
assert_syntax_error "-170141183460469231731687303715884105729", "-170141183460469231731687303715884105729 doesn't fit in an Int64"
assert_syntax_error "-999999999999999999999999999999999999999", "-999999999999999999999999999999999999999 doesn't fit in an Int64"

assert_syntax_error "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF doesn't fit in an UInt64"
assert_syntax_error "0o7777777777777777777777777777777777777777777777777", "0o7777777777777777777777777777777777777777777777777 doesn't fit in an UInt64"
assert_syntax_error "-0o7777777777777777777777777777777777777777777777777", "-0o7777777777777777777777777777777777777777777777777 doesn't fit in an Int64"

it_lexes_number :i128, ["9223372036854775808_i128", "9223372036854775808"]
it_lexes_number :i128, ["-9223372036854775809_i128", "-9223372036854775809"]
it_lexes_number :u128, ["118446744073709551616_u128", "118446744073709551616"]
it_lexes_number :u128, ["18446744073709551616_u128", "18446744073709551616"]
it_lexes_number :i128, ["170141183460469231731687303715884105727_i128", "170141183460469231731687303715884105727"]
it_lexes_number :u128, ["170141183460469231731687303715884105728_u128", "170141183460469231731687303715884105728"]
it_lexes_number :u128, ["340282366920938463463374607431768211455_u128", "340282366920938463463374607431768211455"]
it_lexes_number :u128, ["0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF_u128", "340282366920938463463374607431768211455"]
it_lexes_number :i128, ["-0x80000000000000000000000000000000_i128", "-170141183460469231731687303715884105728"]
assert_syntax_error "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF doesn't fit in an UInt64, try using the suffix u128"
assert_syntax_error "-0x80000000000000000000000000000000", "-0x80000000000000000000000000000000 doesn't fit in an Int64, try using the suffix i128"
assert_syntax_error "-0x80000000000000000000000000000001", "-0x80000000000000000000000000000001 doesn't fit in an Int64"
assert_syntax_error "-1_u128", "Invalid negative value -1 for UInt128"

assert_syntax_error "1__1", "consecutive underscores in numbers aren't allowed"
Expand Down Expand Up @@ -351,17 +368,23 @@ describe "Lexer" do
it_lexes_u64 [["0xffff_ffff_ffff_ffff", "18446744073709551615"], ["0o177777_77777777_77777777", "18446744073709551615"], ["0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111", "18446744073709551615"]]
it_lexes_u64 [["0x00ffffffffffffffff", "18446744073709551615"], ["0o001777777777777777777777", "18446744073709551615"], ["0b001111111111111111111111111111111111111111111111111111111111111111", "18446744073709551615"]]
# 2**64
assert_syntax_error "0x10000_0000_0000_0000", "0x10000_0000_0000_0000 doesn't fit in an UInt64"
assert_syntax_error "0x10000_0000_0000_0000", "0x10000_0000_0000_0000 doesn't fit in an UInt64, try using the suffix i128"
it_lexes_number :i128, ["0x10000_0000_0000_0000_i128", "18446744073709551616"]
assert_syntax_error "0x10000_0000_0000_0000_u64", "0x10000_0000_0000_0000 doesn't fit in an UInt64"
assert_syntax_error "0xfffffffffffffffff_u64", "0xfffffffffffffffff doesn't fit in an UInt64"
assert_syntax_error "0o200000_00000000_00000000", "0o200000_00000000_00000000 doesn't fit in an UInt64"
assert_syntax_error "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000", "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000 doesn't fit in an UInt64"
assert_syntax_error "0o200000_00000000_00000000_u64", "0o200000_00000000_00000000 doesn't fit in an UInt64"
assert_syntax_error "0o200000_00000000_00000000", "0o200000_00000000_00000000 doesn't fit in an UInt64, try using the suffix i128"
assert_syntax_error "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_u64", "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000 doesn't fit in an UInt64"
assert_syntax_error "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000", "0b100000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000 doesn't fit in an UInt64, try using the suffix i128"
# Very large
assert_syntax_error "0x1afafafafafafafafafafaf", "0x1afafafafafafafafafafaf doesn't fit in an UInt64"
assert_syntax_error "0x1afafafafafafafafafafaf", "0x1afafafafafafafafafafaf doesn't fit in an UInt64, try using the suffix i128"
assert_syntax_error "0x1afafafafafafafafafafafu64", "0x1afafafafafafafafafafaf doesn't fit in an UInt64"
assert_syntax_error "0x1afafafafafafafafafafafi32", "0x1afafafafafafafafafafaf doesn't fit in an Int32"
assert_syntax_error "0o1234567123456712345671234567", "0o1234567123456712345671234567 doesn't fit in an UInt64"
assert_syntax_error "0o1234567123456712345671234567u64", "0o1234567123456712345671234567 doesn't fit in an UInt64"
assert_syntax_error "0o1234567123456712345671234567", "0o1234567123456712345671234567 doesn't fit in an UInt64, try using the suffix i128"
assert_syntax_error "0o12345671234567_12345671234567_i8", "0o12345671234567_12345671234567 doesn't fit in an Int8"
assert_syntax_error "0b100000000000000000000000000000000000000000000000000000000000000000", "0b100000000000000000000000000000000000000000000000000000000000000000 doesn't fit in an UInt64"
assert_syntax_error "0b100000000000000000000000000000000000000000000000000000000000000000", "0b100000000000000000000000000000000000000000000000000000000000000000 doesn't fit in an UInt64, try using the suffix i128"
assert_syntax_error "0b100000000000000000000000000000000000000000000000000000000000000000u64", "0b100000000000000000000000000000000000000000000000000000000000000000 doesn't fit in an UInt64"

it_lexes_i64 [["0o700000000000000000000", "8070450532247928832"]]
it_lexes_u64 [["0o1000000000000000000000", "9223372036854775808"]]
Expand Down
6 changes: 2 additions & 4 deletions src/compiler/crystal/codegen/codegen.cr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,11 +475,9 @@ module Crystal
when :u64
@last = int64(node.value.to_u64)
when :i128
# TODO: implement String#to_i128 and use it
@last = int128(node.value.to_i64)
@last = int128(node.value.to_i128)
when :u128
# TODO: implement String#to_u128 and use it
@last = int128(node.value.to_u64)
@last = int128(node.value.to_u128)
when :f32
@last = float32(node.value)
when :f64
Expand Down
38 changes: 25 additions & 13 deletions src/compiler/crystal/syntax/lexer.cr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1450,24 +1450,24 @@ module Crystal
@token.raw = ":#{value}" if @wants_raw
end

macro gen_check_int_fits_in_size(type, method, size, number_size, raw_number_string, start, pos_before_suffix, negative, actual_type = nil)
macro gen_check_int_fits_in_size(type, method, size, number_size, raw_number_string, start, pos_before_suffix, negative)
{% if type.stringify.starts_with? "U" %}
raise "Invalid negative value #{string_range({{start}}, {{pos_before_suffix}})} for {{type}}", @token, (current_pos - {{start}}) if {{negative}}
{% end %}

if !@token.value || {{number_size}} > {{size}} || ({{number_size}} == {{size}} && {{raw_number_string}}.to_{{method.id}}? == nil)
{% if actual_type.nil? %}
raise_value_doesnt_fit_in "{{type}}", {{start}}, {{pos_before_suffix}}
{% else %}
raise("#{string_range({{start}}, {{pos_before_suffix}})} doesn't fit in an {{actual_type}}. {{type}} literals that don't fit in an {{actual_type}} are currently not supported", @token, current_pos - {{start}})
{% end %}
raise_value_doesnt_fit_in "{{type}}", {{start}}, {{pos_before_suffix}}
end
end

def raise_value_doesnt_fit_in(type, start, pos_before_suffix)
raise "#{string_range(start, pos_before_suffix)} doesn't fit in an #{type}", @token, (current_pos - start)
end

def raise_value_doesnt_fit_in(type, start, pos_before_suffix, alternative)
raise "#{string_range(start, pos_before_suffix)} doesn't fit in an #{type}, try using the suffix #{alternative}", @token, (current_pos - start)
end

private def scan_number(start, negative = false)
@token.type = :NUMBER
base = 10
Expand Down Expand Up @@ -1548,7 +1548,10 @@ module Crystal
raw_number_string = raw_number_string.delete('_') if has_underscores
@token.value = raw_number_string
else
# The conversion to base 10 is first tried using a UInt64 to circumvent compiler
# regressions caused by bugs in the platform's UInt128 implementation.
base10_number_string = raw_number_string.to_u64?(base: base, underscore: true).try &.to_s
base10_number_string ||= raw_number_string.to_u128?(base: base, underscore: true).try &.to_s
if base10_number_string
number_size = base10_number_string.size
first_byte = @reader.string.byte_at(start).chr
Expand All @@ -1574,17 +1577,26 @@ module Crystal
if raw_number_string.to_i64?
:i64
elsif negative
raise_value_doesnt_fit_in(Int64, start, pos_before_suffix)
raise_value_doesnt_fit_in(Int64, start, pos_before_suffix, "i128")
else
:u64
end
when 20
raise_value_doesnt_fit_in(Int64, start, pos_before_suffix) if negative
raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix) unless raw_number_string.to_u64?
raise_value_doesnt_fit_in(Int64, start, pos_before_suffix, "i128") if negative
raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix, "i128") unless raw_number_string.to_u64?
:u64
when 21..38
raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix, "i128")
when 39
if raw_number_string.to_i128?
raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix, "i128")
elsif raw_number_string.to_u128?
raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix, "u128")
else
raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix)
end
else
raise_value_doesnt_fit_in(Int64, start, pos_before_suffix) if negative
raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix)
raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix)
end
else
case @token.number_kind
Expand All @@ -1596,8 +1608,8 @@ module Crystal
when :u32 then gen_check_int_fits_in_size(UInt32, :u32, 10, number_size, raw_number_string, start, pos_before_suffix, negative)
when :i64 then gen_check_int_fits_in_size(Int64, :i64, 19, number_size, raw_number_string, start, pos_before_suffix, negative)
when :u64 then gen_check_int_fits_in_size(UInt64, :u64, 20, number_size, raw_number_string, start, pos_before_suffix, negative)
when :i128 then gen_check_int_fits_in_size(Int128, :i64, 19, number_size, raw_number_string, start, pos_before_suffix, negative, Int64)
when :u128 then gen_check_int_fits_in_size(UInt128, :u64, 20, number_size, raw_number_string, start, pos_before_suffix, negative, UInt64)
when :i128 then gen_check_int_fits_in_size(Int128, :i128, 39, number_size, raw_number_string, start, pos_before_suffix, negative)
when :u128 then gen_check_int_fits_in_size(UInt128, :u128, 39, number_size, raw_number_string, start, pos_before_suffix, negative)
end
end
end
Expand Down

0 comments on commit bfddd26

Please sign in to comment.