Fix line counting in token and tokens functions

The line counter becomes invalid, when the rules with linewrap are
used. This issue appears, because the parsing FSM does not rollback
the line counter after attempting such rule.
Unit tests for 'token' and 'tokens' are also added.

lib/parsetools/include/leexinc.hrl

@@ -105,8 +105,8 @@ token(S0, Ics0, L0, Tcs, Tlen0, Tline, A0, Alen0) ->
         {reject,_Alen1,Tlen1,Ics1,L1,_S1} ->    % No token match
             Error = {Tline,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}},
             {done,{error,Error,L1},Ics1};
-        {A1,Alen1,_Tlen1,_Ics1,L1,_S1} ->       % Use last accept match
-            token_cont(yysuf(Tcs, Alen1), L1, yyaction(A1, Alen1, Tcs, Tline))
+        {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} ->       % Use last accept match
+            token_cont(yysuf(Tcs, Alen1), L0, yyaction(A1, Alen1, Tcs, Tline))
     end.
 
 %% token_cont(RestChars, Line, Token)
@@ -177,9 +177,9 @@ tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Ts, A0, Alen0) ->
             %% Skip rest of tokens.
             Error = {L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}},
             skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error);
-        {A1,Alen1,_Tlen1,_Ics1,L1,_S1} ->
+        {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} ->
             Token = yyaction(A1, Alen1, Tcs, Tline),
-            tokens_cont(yysuf(Tcs, Alen1), L1, Token, Ts)
+            tokens_cont(yysuf(Tcs, Alen1), L0, Token, Ts)
     end.
 
 %% tokens_cont(RestChars, Line, Token, Tokens)

lib/parsetools/test/leex_SUITE.erl

@@ -888,15 +888,28 @@ Erlang code.
     XrlFile = filename:join(Dir, "test_line_wrap.xrl"),
     ?line ok = file:write_file(XrlFile, Xrl),
     ErlFile = filename:join(Dir, "test_line_wrap.erl"),
-    ?line {ok, _} = leex:file(XrlFile, []),
-    ?line {ok, _} = compile:file(ErlFile, [{outdir,Dir}]),
+    {ok, _} = leex:file(XrlFile, []),
+    {ok, _} = compile:file(ErlFile, [{outdir,Dir}]),
     code:purge(test_line_wrap),
     AbsFile = filename:rootname(ErlFile, ".erl"),
     code:load_abs(AbsFile, test_line_wrap),
     fun() ->
             S = "aaa\naaa",
             {ok,[{second,1},{second,2}],2} = test_line_wrap:string(S)
     end(),
+    fun() ->
+            S = "aaa\naaa",
+            {ok,[{second,3},{second,4}],4} = test_line_wrap:string(S, 3)
+    end(),
+    fun() ->
+            {done,{ok,{second,1},1},"\na"} = test_line_wrap:token([], "a\na"),
+            {more,Cont1} = test_line_wrap:token([], "\na"),
+            {done,{ok,{second,2},2},eof} = test_line_wrap:token(Cont1, eof)
+    end(),
+    fun() ->
+            {more,Cont1} = test_line_wrap:tokens([], "a\na"),
+            {done,{ok,[{second,1},{second,2}],2},eof} = test_line_wrap:tokens(Cont1, eof)
+    end(),
     ok.
 
 %% End of line_wrap