Fixed some_string_in_re once more

Implement blockwise leading variable split (good speed-up on word equation solving with large constant sequences)

src/ast/rewriter/seq_rewriter.cpp

@@ -5388,21 +5388,91 @@ lbool seq_rewriter::some_string_in_re(expr* r, zstring& s) {
 struct re_eval_pos {
     expr_ref e; // use reference to avoid gc
     unsigned str_len;
-    buffer<std::pair<unsigned, unsigned>> exclude;
+    char_set allowed;      // characters still possible for the position under decision
     bool needs_derivation;
 };
 
+/**
+   Set of characters satisfying the condition of a symbolic derivative's ite.
+   The condition constrains the single derivative variable; returns false for
+   conditions outside the supported (boolean combination of bounds) fragment.
+*/
+bool seq_rewriter::char_set_of_condition(expr* e, char_set& result) {
+    const unsigned max_c = u().max_char();
+    auto is_deriv_var = [](expr* v) { return is_var(v) && to_var(v)->get_idx() == 0; };
+    expr* x = nullptr, * y = nullptr, * a = nullptr;
+    unsigned ch = 0;
+    if (m().is_true(e)) {
+        result = char_set::full(max_c);
+        return true;
+    }
+    if (m().is_false(e)) {
+        result = char_set();
+        return true;
+    }
+    if (m().is_not(e, a)) {
+        char_set s;
+        if (!char_set_of_condition(a, s))
+            return false;
+        result = s.complement(max_c);
+        return true;
+    }
+    if (m().is_and(e)) {
+        result = char_set::full(max_c);
+        for (expr* arg : *to_app(e)) {
+            char_set s;
+            if (!char_set_of_condition(arg, s))
+                return false;
+            result = result.intersect_with(s);
+        }
+        return true;
+    }
+    if (m().is_or(e)) {
+        result = char_set();
+        for (expr* arg : *to_app(e)) {
+            char_set s;
+            if (!char_set_of_condition(arg, s))
+                return false;
+            result.add(s);
+        }
+        return true;
+    }
+    if (m_util.is_char_le(e, x, y)) {
+        if (m_util.is_const_char(x, ch) && is_deriv_var(y)) {
+            result = ch > max_c ? char_set() : char_set(char_range(ch, max_c + 1));
+            return true;
+        }
+        if (m_util.is_const_char(y, ch) && is_deriv_var(x)) {
+            result = char_set(char_range(0, std::min(ch, max_c) + 1));
+            return true;
+        }
+        return false;
+    }
+    if (m().is_eq(e, x, y)) {
+        if (is_deriv_var(x))
+            std::swap(x, y);
+        if (m_util.is_const_char(x, ch) && is_deriv_var(y)) {
+            result = char_set();
+            if (ch <= max_c)
+                result.add(ch);
+            return true;
+        }
+        return false;
+    }
+    return false;
+}
+
 lbool seq_rewriter::some_string_in_re(expr_mark& visited, expr* r, unsigned_vector& str) {
     SASSERT(str.empty());
+    const unsigned max_c = u().max_char();
     vector<re_eval_pos> todo;
-    todo.push_back({ expr_ref(r, m()), 0, {}, true });
+    todo.push_back({ expr_ref(r, m()), 0, char_set::full(max_c), true });
     while (!todo.empty()) {
         re_eval_pos current = todo.back();
         todo.pop_back();
         r = current.e;
         str.resize(current.str_len);
         if (current.needs_derivation) {
-            SASSERT(current.exclude.empty());
             // We are looking for the next character => generate derivation
             if (visited.is_marked(r))
                 continue;
@@ -5414,72 +5484,52 @@ lbool seq_rewriter::some_string_in_re(expr_mark& visited, expr* r, unsigned_vect
             visited.mark(r);
             if (re().is_union(r)) {
                 for (expr* arg : *to_app(r)) {
-                    todo.push_back({ expr_ref(arg, m()), str.size(), {}, true });
+                    todo.push_back({ expr_ref(arg, m()), str.size(), char_set::full(max_c), true });
                 }
                 continue;
             }
 
             r = mk_derivative(r);
         }
-        // otw. we are still in the process of deciding case of the derivation to take
-
-        buffer<std::pair<unsigned, unsigned>> exclude = std::move(current.exclude);
+        // otw. we are still in the process of deciding case of the derivation to
+        // take.  All ite conditions of a derivative constrain the SAME (not yet
+        // committed) character, so they are collected in `allowed` and the
+        // character is only chosen once a plain regex state is reached.  Deriving
+        // a branch that is itself an ite would (re-)interpret its conditions as
+        // constraints on the *next* character.
+        char_set allowed = std::move(current.allowed);
 
         expr* c, * th, * el;
         if (re().is_empty(r))
             continue;
         if (re().is_union(r)) {
             for (expr* arg : *to_app(r)) {
-                todo.push_back({ expr_ref(arg, m()), str.size(), exclude, false });
+                todo.push_back({ expr_ref(arg, m()), str.size(), allowed.clone(), false });
             }
             continue;
         }
         if (m().is_ite(r, c, th, el)) {
-            unsigned low = 0, high = zstring::unicode_max_char();
-            bool has_bounds = get_bounds(c, low, high);
-            if (!re().is_empty(el)) {
-                if (has_bounds)
-                    exclude.push_back({ low, high });
-                todo.push_back({ expr_ref(el, m()), str.size(), std::move(exclude), false });
-            }
-            if (has_bounds) {
-                // I want this case to be processed first => push it last
-                // reason: current string is only pruned
-                SASSERT(low <= high);
-                str.push_back(low);           // ASSERT: low .. high does not intersect with exclude
-                todo.push_back({ expr_ref(th, m()), str.size(), {}, true });
-            }
+            char_set cond;
+            if (!char_set_of_condition(c, cond))
+                return l_undef;
+            char_set else_allowed = allowed.intersect_with(cond.complement(max_c));
+            char_set then_allowed = allowed.intersect_with(cond);
+            if (!re().is_empty(el) && !else_allowed.is_empty())
+                todo.push_back({ expr_ref(el, m()), str.size(), std::move(else_allowed), false });
+            // I want the then-case to be processed first => push it last
+            if (!re().is_empty(th) && !then_allowed.is_empty())
+                todo.push_back({ expr_ref(th, m()), str.size(), std::move(then_allowed), false });
             continue;
         }
 
         if (is_ground(r)) {
-            // ensure selected character is not in exclude
-            unsigned ch = 'a';
-            bool wrapped = false;
-            bool failed = false;
-            while (true) {
-                bool found = false;
-                for (auto [l, h] : exclude) {
-                    if (l <= ch && ch <= h) {
-                        found = true;
-                        ch = h + 1;
-                    }
-                }
-                if (!found)
-                    break;
-                if (ch != zstring::unicode_max_char() + 1)
-                    continue;
-                if (wrapped) {
-                    failed = true;
-                    break;
-                }
-                ch = 0;
-                wrapped = true;
-            }
-            if (failed)
+            // the ite tree is resolved: commit a character consistent with every
+            // condition along the path
+            if (allowed.is_empty())
                 continue;
+            unsigned ch = allowed.contains('a') ? 'a' : allowed.first_char();
             str.push_back(ch);
-            todo.push_back({ expr_ref(r, m()), str.size(), {}, true });
+            todo.push_back({ expr_ref(r, m()), str.size(), char_set::full(max_c), true });
             continue;
         }
 
@@ -5488,32 +5538,3 @@ lbool seq_rewriter::some_string_in_re(expr_mark& visited, expr* r, unsigned_vect
     return l_false;
 }
 
-bool seq_rewriter::get_bounds(expr* e, unsigned& low, unsigned& high) {
-    low = 0; 
-    high = zstring::unicode_max_char();
-    ptr_buffer<expr> todo;
-    todo.push_back(e);
-    expr* x, * y;
-    unsigned ch = 0;
-    while (!todo.empty()) {
-        e = todo.back();
-        todo.pop_back();
-        if (m().is_and(e)) 
-            todo.append(to_app(e)->get_num_args(), to_app(e)->get_args());  
-        else if (m_util.is_char_le(e, x, y) && m_util.is_const_char(x, ch) && is_var(y))
-            low = std::max(ch, low);
-        else if (m_util.is_char_le(e, x, y) && m_util.is_const_char(y, ch) && is_var(x))
-            high = std::min(ch, high);
-        else if (m().is_eq(e, x, y) && is_var(x) && m_util.is_const_char(y, ch)) {
-            low = std::max(ch, low);
-            high = std::min(ch, high);
-        }
-        else if (m().is_eq(e, x, y) && is_var(y) && m_util.is_const_char(x, ch)) {
-            low = std::max(ch, low);
-            high = std::min(ch, high);
-        }
-        else
-            return false;
-    }
-    return low <= high;
-}

src/ast/rewriter/seq_rewriter.h

@@ -330,7 +330,7 @@ class seq_rewriter {
 
     void intersect(unsigned lo, unsigned hi, svector<std::pair<unsigned, unsigned>>& ranges);
 
-    bool get_bounds(expr* e, unsigned& low, unsigned& high);
+    bool char_set_of_condition(expr* e, char_set& result);
     lbool some_string_in_re(expr_mark& visited, expr* r, unsigned_vector& str);
 
 public:

src/params/smt_params.cpp

@@ -63,6 +63,7 @@ void smt_params::updt_local_params(params_ref const & _p) {
     m_nseq_regex_factorization_threshold = p.nseq_regex_factorization_threshold();
     m_nseq_regex_factorization_eager = p.nseq_regex_factorization_eager();
     m_nseq_regex_dynamic_decomposition = p.nseq_regex_dynamic_decomposition();
+    m_nseq_block_compression = p.nseq_block_compression();
     m_nseq_signature = p.nseq_signature();
     m_nseq_fine_wilf = p.nseq_fine_wilf();
     m_nseq_monadic_split = p.nseq_monadic_split();

src/params/smt_params.h

@@ -258,6 +258,7 @@ struct smt_params : public preprocessor_params,
     unsigned m_nseq_regex_factorization_threshold = 1;
     bool m_nseq_regex_factorization_eager = false;
     bool m_nseq_regex_dynamic_decomposition = true;
+    unsigned m_nseq_block_compression = 4;
     bool m_nseq_signature = false;
     bool m_nseq_fine_wilf = false;
     bool m_nseq_monadic_split = false;

src/params/smt_params_helper.pyg

@@ -142,6 +142,7 @@ def_module_params(module_name='smt',
                           ('nseq.regex_factorization_threshold', UINT, 1, 'maximum number of cases to factor a classical regex into in a single step (gives completeness on classical regexes)'),
                           ('nseq.regex_factorization_eager', BOOL, False, 'apply regex factorization (sigma splitting) eagerly in the theory interface (propagate_pos_mem) instead of lazily inside the Nielsen graph'),
                           ('nseq.regex_dynamic_decomposition', BOOL, True, 'decompose cyles detected by unwinding regexes'),
+                          ('nseq.block_compression', UINT, 1, 'in the char-vs-var Nielsen modifier, the maximum number of leading single-character tokens the variable is split against in one step, instead of one character at a time'),
                           ('nseq.signature', BOOL, False, 'enable heuristic signature-based string equation splitting in Nielsen solver'),
                           ('nseq.fine_wilf', BOOL, False, 'enable Fine & Wilf overlap splitting for equations with different-base power heads in the Nielsen solver (breaks the divergent one-copy peel loop)'),
                           ('nseq.monadic_split', BOOL, False, 'enable the continuation-regex intersection modifier (seq_monadic) in the Nielsen solver: closes a node when several memberships on the same sequence have a provably empty language intersection'),

src/smt/seq/seq_nielsen.cpp

@@ -3239,6 +3239,31 @@ namespace seq {
         return false;
     }
 
+    void nielsen_graph::leading_char_block(euf::snode const* side, const bool fwd,
+                                           const unsigned cap, euf::snode_vector& out) {
+        euf::snode_vector toks;
+        collect_tokens_dir(side, fwd, toks);
+        out.reset();
+        for (unsigned i = 0; i < toks.size() && out.size() < cap; i++) {
+            // Single-character tokens only.  Symbolic units qualify: the case
+            // split below needs each token's LENGTH (1), not its value.
+            if (!toks[i]->is_char_or_unit())
+                break;
+            out.push_back(toks[i]);
+        }
+    }
+
+    euf::snode const* nielsen_graph::mk_block_word(euf::snode_vector const& block, const unsigned k,
+                                                   const bool fwd, sort* s) {
+        if (k == 0)
+            return m_sg.mk_empty_seq(s);
+        euf::snode const* w = nullptr;
+        for (unsigned i = 0; i < k; i++) {
+            w = dir_concat(m_sg, w, block[i], fwd);   // grows in DIRECTION order
+        }
+        return w;
+    }
+
     bool nielsen_graph::apply_const_nielsen(nielsen_node* node) {
         for (str_eq const& eq : node->str_eqs()) {
             if (eq.is_trivial())
@@ -3254,24 +3279,72 @@ namespace seq {
                 // char vs var: branch 1: var -> ε, branch 2: var -> char·var   (depending on direction)
                 euf::snode const* char_head = lhead->is_char_or_unit() ? lhead : (rhead->is_char_or_unit() ? rhead : nullptr);
                 euf::snode const* var_head = lhead->is_var() ? lhead : (rhead->is_var() ? rhead : nullptr);
-                if (char_head && var_head) {
+                if (!char_head || !var_head)
+                    continue;
+
+                const euf::snode* const_side = (lhead == char_head) ? eq.m_lhs : eq.m_rhs;
+                const euf::snode* var_side   = (lhead == char_head) ? eq.m_rhs : eq.m_lhs;
+                euf::snode_vector block;
+                if (m_block_compression > 1)
+                    leading_char_block(const_side, fwd, m_block_compression, block);
+                else
+                    block.push_back(char_head);
+                const unsigned m_len = block.size();
+                SASSERT(m_len >= 1);
+                if (m_len > 1) {
+                    ++m_stats.m_mod_block_compression;
+                    m_stats.m_block_chars_consumed += m_len;
+                }
+
+                // Token following x on the variable side (null if x is the whole side)
+                euf::snode_vector var_toks;
+                collect_tokens_dir(var_side, fwd, var_toks);
+                SASSERT(var_toks.empty() || var_toks[0] == var_head);
+                euf::snode const* const next_tok = var_toks.size() > 1 ? var_toks[1] : nullptr;
+
+                sort* const seq_sort = var_head->get_sort();
+
+                
+                // x = ε  (the classic "var → ε" branch; a ground, eliminating
+                // substitution, so it keeps its progress flag)
+                {
                     nielsen_node* child = mk_child(node);
                     nielsen_edge* e = mk_edge(node, child, "nielsen const 0", true);
-                    const nielsen_subst s1(var_head, m_sg.mk_empty_seq(var_head->get_sort()), eq.m_dep);
-                    e->add_subst(s1);
-                    child->apply_subst(m_sg, s1);
-
-
-                    euf::snode const* tail = get_tail(var_head, a.mk_int(1), fwd);
-                    euf::snode const* replacement = dir_concat(m_sg, char_head, tail, fwd);
-                    child = mk_child(node);
-                    e = mk_edge(node, child, "nielsen const &gt;", false);
-                    e->add_side_constraint(mk_constraint(a.mk_ge(compute_length_expr(tail), a.mk_int(0)), eq.m_dep));
-                    const nielsen_subst s2(var_head, replacement, eq.m_dep);
-                    e->add_subst(s2);
-                    child->apply_subst(m_sg, s2);
-                    return true;
+                    const nielsen_subst s(var_head, m_sg.mk_empty_seq(seq_sort), eq.m_dep);
+                    e->add_subst(s);
+                    child->apply_subst(m_sg, s);
                 }
+
+                // x = w · x'  —  the whole block is consumed in one step and
+                // cancels against the other side during simplification.  For
+                // m_len = 1 this is exactly the classic one-character peel.
+                {
+                    euf::snode const* tail = get_tail(var_head, a.mk_int(m_len), fwd);
+                    euf::snode const* replacement =
+                        dir_concat(m_sg, mk_block_word(block, m_len, fwd, seq_sort), tail, fwd);
+                    nielsen_node* child = mk_child(node);
+                    nielsen_edge* e = mk_edge(node, child, "nielsen const &gt;", false);
+                    e->add_side_constraint(mk_constraint(a.mk_ge(compute_length_expr(tail), a.mk_int(0)), eq.m_dep));
+                    const nielsen_subst s(var_head, replacement, eq.m_dep);
+                    e->add_subst(s);
+                    child->apply_subst(m_sg, s);
+                }
+
+                // x = w[0..k) for 0 < k < m_len: x ends strictly inside the block.
+                for (unsigned k = 1; k < m_len; ++k) {
+                    // Clash lookahead [maybe drop it at the other position]
+                    if (!next_tok ||
+                        (next_tok->is_char() && block[k]->is_char() && next_tok->id() != block[k]->id())) {
+                        ++m_stats.m_block_children_pruned;
+                        continue;
+                    }
+                    nielsen_node* child = mk_child(node);
+                    nielsen_edge* e = mk_edge(node, child, "nielsen block =", false);
+                    const nielsen_subst s(var_head, mk_block_word(block, k, fwd, seq_sort), eq.m_dep);
+                    e->add_subst(s);
+                    child->apply_subst(m_sg, s);
+                }
+                return true;
             }
         }
         return false;
@@ -7004,6 +7077,9 @@ namespace seq {
         st.update("nseq mod regex fact",       m_stats.m_mod_regex_factorization);
         st.update("nseq mod monadic split",    m_stats.m_mod_monadic_split);
         st.update("nseq mod const nielsen",    m_stats.m_mod_const_nielsen);
+        st.update("nseq mod block compr",      m_stats.m_mod_block_compression);
+        st.update("nseq block chars",          m_stats.m_block_chars_consumed);
+        st.update("nseq block pruned",         m_stats.m_block_children_pruned);
         st.update("nseq mod signature split",  m_stats.m_mod_signature_split);
         st.update("nseq mod regex var",        m_stats.m_mod_regex_var_split);
         st.update("nseq mod regex if",         m_stats.m_mod_regex_if_split);

src/smt/seq/seq_nielsen.h

@@ -201,8 +201,8 @@ namespace seq {
 
         bool operator<(const str_eq& other) const {
             if (m_lhs != other.m_lhs)
-                return m_lhs < other.m_lhs;
-            return m_rhs < other.m_rhs;
+                return m_lhs->id() < other.m_lhs->id();
+            return m_rhs->id() < other.m_rhs->id();
         }
 
         unsigned hash() const {
@@ -260,9 +260,10 @@ namespace seq {
         }
 
         bool operator<(const str_deq& other) const {
+            // by snode ID, not address
             if (m_lhs != other.m_lhs)
-                return m_lhs < other.m_lhs;
-            return m_rhs < other.m_rhs;
+                return m_lhs->id() < other.m_lhs->id();
+            return m_rhs->id() < other.m_rhs->id();
         }
 
         unsigned hash() const {
@@ -355,16 +356,18 @@ namespace seq {
             // Regex-factorization heuristic: order primarily by the estimated
             // automaton size of the regex so that the cheapest membership is
             // factorized first (see snode::regex_weight / sgraph::compute_regex_weight).
-            // The remaining pointer comparisons are just deterministic tie-breakers
-            // to keep this a total order for canonicalization/sorting.
+            // The remaining comparisons are tie-breakers keeping this a total
+            // order for canonicalization/sorting.  They compare snode IDs, NOT
+            // snode addresses: addresses vary between runs, which would make the
+            // canonical order (and hence every node hash) run-dependent.
             const unsigned w1 = m_regex->regex_weight();
             const unsigned w2 = other.m_regex->regex_weight();
             if (w1 != w2)
                 return w1 < w2;
             if (m_str != other.m_str)
-                return m_str < other.m_str;
+                return m_str->id() < other.m_str->id();
             if (m_regex != other.m_regex)
-                return m_regex < other.m_regex;
+                return m_regex->id() < other.m_regex->id();
             // View annotation tie-breakers, keeping the order consistent with
             // operator==: two views on the same (str, state) that differ only
             // in kind/root/ν must not compare equivalent, otherwise std::sort
@@ -869,6 +872,9 @@ namespace seq {
         unsigned m_mod_regex_factorization = 0;
         unsigned m_mod_monadic_split   = 0;
         unsigned m_mod_const_nielsen   = 0;
+        unsigned m_mod_block_compression = 0;
+        unsigned m_block_chars_consumed  = 0;
+        unsigned m_block_children_pruned = 0;
         unsigned m_mod_regex_var_split = 0;
         unsigned m_mod_signature_split = 0;
         unsigned m_mod_power_split     = 0;
@@ -944,6 +950,7 @@ namespace seq {
         unsigned                      m_max_nodes = 0;          // 0 = unlimited
         bool                          m_parikh_enabled = true;
         bool                          m_signature_split = false;
+        unsigned                      m_block_compression = 4;
         bool                          m_fine_wilf = false;
         bool                          m_monadic_split = false;
         unsigned                      m_regex_factorization_threshold = 1;
@@ -1195,6 +1202,8 @@ namespace seq {
 
         void set_signature_split(bool e) { m_signature_split = e; }
 
+        void set_block_compression(unsigned cap) { m_block_compression = cap; }
+
         void set_fine_wilf(bool e) { m_fine_wilf = e; }
 
         void set_monadic_split(bool e) { m_monadic_split = e; }
@@ -1551,9 +1560,19 @@ namespace seq {
         // deterministic modifier: var = ε, same-head cancel
         bool apply_det_modifier(nielsen_node* node);
 
-        // const nielsen modifier: char vs var (2 branches per case)
+        // const nielsen modifier: char vs var (usually 2 branches)
         bool apply_const_nielsen(nielsen_node* node);
 
+        // maximal prefix of single-character tokens at the directional head of
+        // `side`, truncated to `cap`.
+        static void leading_char_block(euf::snode const* side, bool fwd, unsigned cap,
+                                       euf::snode_vector& out);
+
+        // the word formed by the first `k` tokens of `block` in direction order
+        // (ε for k = 0); `s` supplies the sequence sort
+        euf::snode const* mk_block_word(euf::snode_vector const& block, unsigned k,
+                                        bool fwd, sort* s);
+
         // variable Nielsen modifier: var vs var, all progress (3 branches)
         bool apply_var_nielsen(nielsen_node* node);
 

src/smt/seq_model.cpp

@@ -632,11 +632,17 @@ namespace smt {
                 visited.insert(d->id());
 
                 // Representative character of the minterm (must lie in mt so
-                // that δ_ch(st) = d).  Minterms over the cycle alphabet are
-                // ranges / to_re singletons / full_char.
+                // that δ_ch(st) = d).  Minterms are ranges / to_re singletons /
+                // full_char, or a UNION of ranges when the partition class has
+                // several intervals — pick the same representative as
+                // brzozowski_deriv (which descends into arg(0) of a union),
+                // otherwise the emitted character does not take the transition
+                // whose derivative `d` we just followed.
                 unsigned ch = 0;
                 expr* me = mt->get_expr();
                 expr* lo = nullptr, *hi = nullptr, *s = nullptr;
+                while (m_seq.re.is_union(me))
+                    me = to_app(me)->get_arg(0);
                 if (m_seq.re.is_range(me, lo, hi))
                     m_sg.decode_re_char(lo, ch);
                 else if (m_seq.re.is_to_re(me, s))

src/smt/theory_nseq.cpp

@@ -1019,6 +1019,7 @@ namespace smt {
                 m_nielsen.set_max_nodes(get_fparams().m_nseq_max_nodes);
                 m_nielsen.set_parikh_enabled(get_fparams().m_nseq_parikh);
                 m_nielsen.set_signature_split(get_fparams().m_nseq_signature);
+                m_nielsen.set_block_compression(get_fparams().m_nseq_block_compression);
                 m_nielsen.set_fine_wilf(get_fparams().m_nseq_fine_wilf);
                 m_nielsen.set_monadic_split(get_fparams().m_nseq_monadic_split);
                 m_nielsen.set_regex_factorization_threshold(get_fparams().m_nseq_regex_factorization_threshold);