Skip to content

Commit

Permalink
Squash merge transliterator-ir
Browse files Browse the repository at this point in the history 
commit 9d55038
Author: Niels Saurer <[email protected]>
Date:   Thu Aug 10 02:03:34 2023 +0200

    fix push_front/push_back mixup

commit dc8dda7
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 23:02:10 2023 +0200

    remove empty line

commit bfe5827
Merge: c85e861 f549131
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 20:57:11 2023 +0200

    Merge branch 'main' into transliterator-ir

commit c85e861
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 17:40:53 2023 +0200

    borrow SingleID

commit 06425a1
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 17:22:03 2023 +0200

    fix comment indentation

commit 2f70922
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 17:09:13 2023 +0200

    update comments

commit 47444ee
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 17:06:43 2023 +0200

    fmt

commit c0de3a0
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 17:03:19 2023 +0200

    fix clippy, allow testing of intermediate pass1 values

commit 227f738
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 16:55:53 2023 +0200

    fix compile errors by introducing 2 small clones per transliterator

commit 512b158
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 16:49:01 2023 +0200

    doesn't compile - missing self deconstruction

commit 7848f09
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 16:40:51 2023 +0200

    use rule group aggregation in pass1

commit 93663e4
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 16:09:29 2023 +0200

    add rule group aggregation

commit 57666eb
Author: Niels Saurer <[email protected]>
Date:   Wed Aug 9 14:12:19 2023 +0200

    Squash of transliterator-compiler

    commit d1812b4
    Author: Niels Saurer <[email protected]>
    Date:   Wed Aug 9 13:31:53 2023 +0200

        fix merge mistake

    commit f15f6eb
    Merge: abb91cc a39cfed
    Author: Niels Saurer <[email protected]>
    Date:   Wed Aug 9 13:27:08 2023 +0200

        Merge branch 'main' into transliterator-compiler

    commit abb91cc
    Author: Niels Saurer <[email protected]>
    Date:   Wed Aug 9 01:12:13 2023 +0200

        reformat tests

    commit f6a10f5
    Author: Niels Saurer <[email protected]>
    Date:   Wed Aug 9 00:30:09 2023 +0200

        sizes => counts

    commit 9ffc2f0
    Author: Niels Saurer <[email protected]>
    Date:   Wed Aug 9 00:26:27 2023 +0200

        add more docs

    commit eae5748
    Author: Niels Saurer <[email protected]>
    Date:   Tue Aug 8 23:46:20 2023 +0200

        remove TODO

    commit 6b09689
    Author: Niels Saurer <[email protected]>
    Date:   Tue Aug 8 23:28:42 2023 +0200

        improve docs

    commit c9b16d5
    Author: Niels Saurer <[email protected]>
    Date:   Tue Aug 8 23:15:23 2023 +0200

        clippy

    commit 020a677
    Author: Niels Saurer <[email protected]>
    Date:   Tue Aug 8 22:53:14 2023 +0200

        add result aggregation to first pass

    commit 2d1bfd7
    Author: Niels Saurer <[email protected]>
    Date:   Tue Aug 8 16:28:23 2023 +0200

        add tests

    commit 6f35ea5
    Author: Niels Saurer <[email protected]>
    Date:   Mon Aug 7 22:25:56 2023 +0200

        CI fixes

    commit c6c4844
    Author: Niels Saurer <[email protected]>
    Date:   Sun Aug 6 20:06:31 2023 +0200

        first steps

    commit fb68218
    Author: Niels Saurer <[email protected]>
    Date:   Wed Jul 19 16:21:33 2023 +0000

        Squash transliterator-parser

        structure for transliterator parser

        start parsing ':: ... ;' rules

        complete ::-rule parsing

        add more global filter tests

        add negative tests for '::'-rules, be more restrictive

        update error docs

        add comment about static UnicodeSet type alias

        add variable defs

        escaping and fix unicodeset handling

        fix unicodeset tests

        function calls

        add variable-inside-unicodesets

        update tests

        rewrite parse_section using parse_element

        fix unquoted literal handling

        add cursor/placeholder tests

        add cursor support

        add allow(unused) for this PR

        remove unused dependencies

        add todo about inefficient unicodeset variablemap handling

        allow usage of UnicodeSet's VariableMap directly in TransliteratorParser

        avoid one allocation per parsed unicodeset

        remove done todo about allocation-free unicodeset parser hook

        avoid allocations for number parsing

        invalid num err with offset

        update comment

        switch to allocation free hex parsing (and support for multi escapes)

        fix main merge conflict

        support \p unicodesets

        remove todo for \p unicodeset parsing

        turn low-prio todo about avoiding clones into note

        turn non-memory-safety safety comments into regular comments

        add issue number to TODOs

        add transliteration component crate
  • Loading branch information
@skius
skius committed Aug 10, 2023
1 parent 208abd7 commit 1145a17
Showing 1 changed file with 7 additions and 8 deletions.
15 changes: 7 additions & 8 deletions experimental/transliterator_parser/src/compile/rule_group_agg.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,6 @@ impl<'p> ForwardRuleGroupAggregator<'p> {

// Represents a non-empty rule group for the forward direction.
#[derive(Debug, Clone)]

enum ForwardRuleGroup<'p> {
Conversion(Vec<UniConversionRule<'p>>),
Transform(Vec<Cow<'p, parse::SingleId>>),
Expand Down Expand Up @@ -183,7 +182,7 @@ impl<'p> ForwardRuleGroup<'p> {
// contiguous C's keep the source order, but contiguous T's are reversed. Also the overall order
// is reversed, of course.
//
// We do this by using VecDeque, push_back, and make_contiguous in the end.
// We do this by using VecDeque, push_front, and make_contiguous in the end.
#[derive(Debug, Clone)]
pub(crate) struct ReverseRuleGroupAggregator<'p> {
current: ReverseRuleGroup<'p>,
Expand Down Expand Up @@ -265,7 +264,7 @@ impl<'p> ReverseRuleGroupAggregator<'p> {
};
let vec_transform_group = transform_group.into(); // non-allocating conversion
self.groups
.push_back((vec_transform_group, associated_conv_group));
.push_front((vec_transform_group, associated_conv_group));
}
}
}
Expand All @@ -278,7 +277,7 @@ impl<'p> ReverseRuleGroupAggregator<'p> {
if let Some(conv_group) = self.preceding_conversion_group.take() {
// a trailing conversion group in source order is the same as having a conversion
// group as the first in-order group. we can just prepend an empty transform group.
self.groups.push_back((Vec::new(), conv_group));
self.groups.push_front((Vec::new(), conv_group));
}

self.groups.into() // non-allocating conversion
Expand All @@ -290,7 +289,7 @@ impl<'p> ReverseRuleGroupAggregator<'p> {
enum ReverseRuleGroup<'p> {
// because contiguous C's are aggregated in source-order, we can just use a Vec
Conversion(Vec<UniConversionRule<'p>>),
// but contiguous T's are aggregated in reverse-order, so we need to use a VecDeque and push_back
// but contiguous T's are aggregated in reverse-order, so we need to use a VecDeque and push_front
Transform(VecDeque<Cow<'p, parse::SingleId>>),
}

Expand All @@ -307,7 +306,7 @@ impl<'p> ReverseRuleGroup<'p> {

fn new_transform(rule: Cow<'p, parse::SingleId>) -> Self {
let mut group = VecDeque::new();
group.push_back(rule);
group.push_front(rule);
Self::Transform(group)
}

Expand All @@ -320,8 +319,8 @@ impl<'p> ReverseRuleGroup<'p> {
}
(Self::Transform(group), UniRule::Transform(rule)) => {
// we receive rules via `push` in source-order, which is the opposite order we want,
// so we push_back.
group.push_back(rule);
// so we push_front.
group.push_front(rule);
None
}
(Self::Conversion(_), UniRule::Transform(new_rule)) => {
Expand Down

0 comments on commit 1145a17

Please sign in to comment.