Skip to content

Commit

Permalink
parsetools: Correct counting of newlines
Browse files Browse the repository at this point in the history 
See https://bugs.erlang.org/browse/ERL-263

The fix in commit c9bc5c9 of PR-431
(erlang#431) introduced new problems.
  • Loading branch information
@uabboli
uabboli committed Sep 29, 2016
1 parent 8bde09c commit 2990325
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 13 deletions.
35 changes: 27 additions & 8 deletions lib/parsetools/include/leexinc.hrl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@ string(Ics0, L0, Tcs, Ts) ->
string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L0), Ts);
{reject,_Alen,Tlen,_Ics1,L1,_S1} -> % After a non-accepting state
{error,{L0,?MODULE,{illegal,yypre(Tcs, Tlen+1)}},L1};
{A,Alen,_Tlen,_Ics1,_L1,_S1} ->
string_cont(yysuf(Tcs, Alen), L0, yyaction(A, Alen, Tcs, L0), Ts)
{A,Alen,Tlen,_Ics1,L1,_S1} ->
Tcs1 = yysuf(Tcs, Alen),
L2 = adjust_line(Tlen, Alen, Tcs1, L1),
string_cont(Tcs1, L2, yyaction(A, Alen, Tcs, L0), Ts)
end.

%% string_cont(RestChars, Line, Token, Tokens)
Expand Down Expand Up @@ -107,8 +109,10 @@ token(S0, Ics0, L0, Tcs, Tlen0, Tline, A0, Alen0) ->
{reject,_Alen1,Tlen1,Ics1,L1,_S1} -> % No token match
Error = {Tline,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}},
{done,{error,Error,L1},Ics1};
{A1,Alen1,_Tlen1,_Ics1,_L1,_S1} -> % Use last accept match
token_cont(yysuf(Tcs, Alen1), L0, yyaction(A1, Alen1, Tcs, Tline))
{A1,Alen1,Tlen1,_Ics1,L1,_S1} -> % Use last accept match
Tcs1 = yysuf(Tcs, Alen1),
L2 = adjust_line(Tlen1, Alen1, Tcs1, L1),
token_cont(Tcs1, L2, yyaction(A1, Alen1, Tcs, Tline))
end.

%% token_cont(RestChars, Line, Token)
Expand Down Expand Up @@ -181,9 +185,11 @@ tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Ts, A0, Alen0) ->
%% Skip rest of tokens.
Error = {L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}},
skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error);
{A1,Alen1,_Tlen1,_Ics1,_L1,_S1} ->
{A1,Alen1,Tlen1,_Ics1,L1,_S1} ->
Token = yyaction(A1, Alen1, Tcs, Tline),
tokens_cont(yysuf(Tcs, Alen1), L0, Token, Ts)
Tcs1 = yysuf(Tcs, Alen1),
L2 = adjust_line(Tlen1, Alen1, Tcs1, L1),
tokens_cont(Tcs1, L2, Token, Ts)
end.

%% tokens_cont(RestChars, Line, Token, Tokens)
Expand Down Expand Up @@ -235,9 +241,11 @@ skip_tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Error, A0, Alen0) ->
{done,{error,Error,L1},eof};
{reject,_Alen1,Tlen1,_Ics1,L1,_S1} ->
skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error);
{A1,Alen1,_Tlen1,_Ics1,L1,_S1} ->
{A1,Alen1,Tlen1,_Ics1,L1,_S1} ->
Token = yyaction(A1, Alen1, Tcs, Tline),
skip_cont(yysuf(Tcs, Alen1), L1, Token, Error)
Tcs1 = yysuf(Tcs, Alen1),
L2 = adjust_line(Tlen1, Alen1, Tcs1, L1),
skip_cont(Tcs1, L2, Token, Error)
end.

%% skip_cont(RestChars, Line, Token, Error)
Expand Down Expand Up @@ -269,6 +277,17 @@ yyrev(List, Tail) -> lists:reverse(List, Tail).
yypre(List, N) -> lists:sublist(List, N).
yysuf(List, N) -> lists:nthtail(N, List).

%% adjust_line(TokenLength, AcceptLength, Chars, Line) -> NewLine
%% Make sure that newlines in Chars are not counted twice.
%% Line has been updated with respect to newlines in the prefix of
%% Chars consisting of (TokenLength - AcceptLength) characters.

adjust_line(N, N, _Cs, L) -> L;
adjust_line(T, A, [$\n|Cs], L) ->
adjust_line(T-1, A, Cs, L-1);
adjust_line(T, A, [_|Cs], L) ->
adjust_line(T-1, A, Cs, L).

%% yystate() -> InitialState.
%% yystate(State, InChars, Line, CurrTokLen, AcceptAction, AcceptLen) ->
%% {Action, AcceptLen, RestChars, Line} |
Expand Down
44 changes: 40 additions & 4 deletions lib/parsetools/test/leex_SUITE.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@

pt/1, man/1, ex/1, ex2/1, not_yet/1,
line_wrap/1,
otp_10302/1, otp_11286/1, unicode/1]).
otp_10302/1, otp_11286/1, unicode/1, otp_13916/1]).

% Default timetrap timeout (set in init_per_testcase).
-define(default_timeout, ?t:minutes(1)).
Expand All @@ -62,12 +62,12 @@ end_per_testcase(_Case, Config) ->
suite() -> [{ct_hooks,[ts_install_cth]}].

all() ->
[{group, checks}, {group, examples}, {group, bugs}].
[{group, checks}, {group, examples}, {group, tickets}, {group, bugs}].

groups() ->
[{checks, [], [file, compile, syntax]},
{examples, [], [pt, man, ex, ex2, not_yet, unicode]},
{tickets, [], [otp_10302, otp_11286]},
{tickets, [], [otp_10302, otp_11286, otp_13916]},
{bugs, [], [line_wrap]}].

init_per_suite(Config) ->
Expand Down Expand Up @@ -1052,7 +1052,7 @@ otp_11286(Config) when is_list(Config) ->
Dir = ?privdir,
UName = [1024] ++ "u",
UDir = filename:join(Dir, UName),
ok = rpc:call(Node, file, make_dir, [UDir]),
_ = rpc:call(Node, file, make_dir, [UDir]),

%% Note: Cannot use UName as filename since the filename is used
%% as module name. To be fixed in R18.
Expand Down Expand Up @@ -1095,6 +1095,42 @@ otp_11286(Config) when is_list(Config) ->
true = test_server:stop_node(Node),
ok.

otp_13916(doc) ->
"OTP-13916. Leex rules with newlines result in bad line numbers";
otp_13916(suite) -> [];
otp_13916(Config) when is_list(Config) ->
Ts = [{otp_13916_1,
<<"Definitions.\n"
"W = [a-zA-Z0-9]\n"
"S = [\\s\\t]\n"
"B = [\\n\\r]\n"
"Rules.\n"
"%% mark line break(s) and empty lines by token 'break'\n"
"%% in order to use as delimiters\n"
"{B}({S}*{B})+ : {token, {break, TokenLine}}.\n"
"{B} : {token, {break, TokenLine}}.\n"
"{S}+ : {token, {blank, TokenLine, TokenChars}}.\n"
"{W}+ : {token, {word, TokenLine, TokenChars}}.\n"
"Erlang code.\n"
"-export([t/0]).\n"
"t() ->\n"
" {ok,[{break,1},{blank,4,\" \"},{word,4,\"breaks\"}],4} =\n"
" string(\"\\n\\n \\n breaks\"),\n"
" {ok,[{break,1},{word,4,\"works\"}],4} =\n"
" string(\"\\n\\n \\nworks\"),\n"
" {ok,[{break,1},{word,4,\"L4\"},{break,4},\n"
" {word,5,\"L5\"},{break,5},{word,7,\"L7\"}], 7} =\n"
" string(\"\\n\\n \\nL4\\nL5\\n\\nL7\"),\n"
" {ok,[{break,1},{blank,4,\" \"},{word,4,\"L4\"},\n"
" {break,4},{blank,5,\" \"},{word,5,\"L5\"},\n"
" {break,5},{blank,7,\" \"},{word,7,\"L7\"}], 7} =\n"
" string(\"\\n\\n \\n L4\\n L5\\n\\n L7\"),\n"
" ok.\n">>,
default,
ok}],
?line run(Config, Ts),
ok.

start_node(Name, Args) ->
[_,Host] = string:tokens(atom_to_list(node()), "@"),
ct:log("Trying to start ~w@~s~n", [Name,Host]),
Expand Down
2 changes: 1 addition & 1 deletion lib/parsetools/test/yecc_SUITE.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2009,7 +2009,7 @@ otp_11286(Config) when is_list(Config) ->
Dir = ?privdir,
UName = [1024] ++ "u",
UDir = filename:join(Dir, UName),
ok = rpc:call(Node, file, make_dir, [UDir]),
_ = rpc:call(Node, file, make_dir, [UDir]),

%% Note: Cannot use UName as filename since the filename is used
%% as module name. To be fixed in R18.
Expand Down

0 comments on commit 2990325

Please sign in to comment.