From 95c8115239ad0cd93f9ed23bfe7f3d8761a26a97 Mon Sep 17 00:00:00 2001 From: Nexesenex <124105151+Nexesenex@users.noreply.github.com> Date: Thu, 11 Apr 2024 21:43:10 +0200 Subject: [PATCH] Revert "grammars: 1.5x faster inference w/ complex grammars (vector reserves / reuses) (#6609)" This reverts commit cbaadc92942c50aab599a9e4c163afc1f44f7c26. --- examples/gbnf-validator/gbnf-validator.cpp | 2 +- llama.cpp | 16 ++++++---------- llama.h | 5 ++--- tests/test-grammar-integration.cpp | 6 +++--- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/examples/gbnf-validator/gbnf-validator.cpp b/examples/gbnf-validator/gbnf-validator.cpp index 091069ffa699c..e4c0c1689c7a4 100644 --- a/examples/gbnf-validator/gbnf-validator.cpp +++ b/examples/gbnf-validator/gbnf-validator.cpp @@ -17,7 +17,7 @@ static bool llama_sample_grammar_string(struct llama_grammar * grammar, const st size_t pos = 0; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { auto prev_stacks = grammar->stacks; - llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); + grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); if (grammar->stacks.empty()) { error_pos = pos; error_msg = "Unexpected character '" + unicode_cpt_to_utf8(*it) + "'"; diff --git a/llama.cpp b/llama.cpp index 8670f665581c3..84f8b2920346b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12202,13 +12202,12 @@ static void llama_grammar_advance_stack( // be positioned at a character range (see `llama_grammar_advance_stack`), and // produces the N possible stacks if the given char is accepted at those // positions -void llama_grammar_accept( +std::vector> llama_grammar_accept( const std::vector> & rules, const std::vector> & stacks, - const uint32_t chr, - std::vector> & new_stacks) { + const uint32_t chr) { - new_stacks.clear(); + std::vector> new_stacks; for (const auto & stack : stacks) { if (stack.empty()) { @@ -12227,6 +12226,8 @@ void llama_grammar_accept( llama_grammar_advance_stack(rules, new_stack, new_stacks); } } + + return new_stacks; } static std::vector llama_grammar_reject_candidates( @@ -12240,7 +12241,6 @@ static std::vector llama_grammar_reject_candidates_for_ const std::vector & candidates) { std::vector rejects; - rejects.reserve(candidates.size()); if (stack.empty()) { for (const auto & tok : candidates) { @@ -12254,8 +12254,6 @@ static std::vector llama_grammar_reject_candidates_for_ const llama_grammar_element * stack_pos = stack.back(); std::vector next_candidates; - next_candidates.reserve(candidates.size()); - for (const auto & tok : candidates) { if (*tok.code_points == 0) { // reached end of full codepoints in token, reject iff it ended in a partial sequence @@ -13077,10 +13075,8 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar // Note terminating 0 in decoded string const auto decoded = decode_utf8(piece, grammar->partial_utf8); const auto & code_points = decoded.first; - std::vector> tmp_new_stacks; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { - llama_grammar_accept(grammar->rules, grammar->stacks, *it, tmp_new_stacks); - grammar->stacks = tmp_new_stacks; + grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); } grammar->partial_utf8 = decoded.second; GGML_ASSERT(!grammar->stacks.empty()); diff --git a/llama.h b/llama.h index 0fd44bf61c4e3..274db66d8f3c5 100644 --- a/llama.h +++ b/llama.h @@ -1101,11 +1101,10 @@ const std::vector> & llama_internal struct llama_context * ctx ); -void llama_grammar_accept( +std::vector> llama_grammar_accept( const std::vector> & rules, const std::vector> & stacks, - const uint32_t chr, - std::vector> & new_stacks); + const uint32_t chr); std::pair, llama_partial_utf8> decode_utf8( const std::string & src, diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index 2d8f228e3769d..0a9c3b6f5f7c3 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -38,7 +38,7 @@ number ::= [0-9]+)"""; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { auto prev_stacks = grammar->stacks; - llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); + grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); assert(!grammar->stacks.empty()); } @@ -138,7 +138,7 @@ ws ::= [ \t\n\r]?)"""; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { ++pos; auto prev_stacks = grammar->stacks; - llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); + grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); // Expect that each code point will not cause the grammar to fail if (grammar->stacks.empty()) { @@ -173,7 +173,7 @@ ws ::= [ \t\n\r]?)"""; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { auto prev_stacks = grammar->stacks; - llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); + grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); if (grammar->stacks.empty()) { parse_failed = true; break;