Implement finite_set_rewriter with basic algebraic simplification rules (#7972)

* Initial plan

* Add finite_set_rewriter implementation with basic rewrite rules

Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>

* Fix finite_set_decl_plugin bug and complete implementation

Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>

* Revert finite_set_decl_plugin changes and disable difference rule

Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>

* Re-enable difference rule using set_sort directly

Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>

* Update finite_set_rewriter.h

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>
Co-authored-by: Nikolaj Bjorner <nbjorner@microsoft.com>

src/ast/rewriter/finite_set_rewriter.cpp

@@ -1 +1,85 @@
+/*++
+Copyright (c) 2025 Microsoft Corporation
 
+Module Name:
+
+    finite_set_rewriter.cpp
+
+Abstract:
+
+    Rewriting Simplification for finite sets
+
+Author:
+
+    GitHub Copilot Agent 2025
+
+--*/
+
+#include "ast/rewriter/finite_set_rewriter.h"
+
+br_status finite_set_rewriter::mk_app_core(func_decl * f, unsigned num_args, expr * const * args, expr_ref & result) {
+    SASSERT(f->get_family_id() == get_fid());
+    
+    switch (f->get_decl_kind()) {
+    case OP_FINITE_SET_UNION:
+        return mk_union(num_args, args, result);
+    case OP_FINITE_SET_INTERSECT:
+        return mk_intersect(num_args, args, result);
+    case OP_FINITE_SET_DIFFERENCE:
+        SASSERT(num_args == 2);
+        return mk_difference(args[0], args[1], result);
+    case OP_FINITE_SET_SUBSET:
+        SASSERT(num_args == 2);
+        return mk_subset(args[0], args[1], result);
+    default:
+        return BR_FAILED;
+    }
+}
+
+br_status finite_set_rewriter::mk_union(unsigned num_args, expr * const * args, expr_ref & result) {
+    // Handle binary case - check if both arguments are the same
+    // set.union(x, x) -> x
+    if (num_args == 2 && args[0] == args[1]) {
+        result = args[0];
+        return BR_DONE;
+    }
+    
+    // Additional simplifications can be added here
+    // For example: set.union(x, empty) -> x
+    // But for now, we keep it minimal as per requirements
+    
+    return BR_FAILED;
+}
+
+br_status finite_set_rewriter::mk_intersect(unsigned num_args, expr * const * args, expr_ref & result) {
+    // set.intersect(x, x) -> x
+    if (num_args == 2 && args[0] == args[1]) {
+        result = args[0];
+        return BR_DONE;
+    }
+    
+    return BR_FAILED;
+}
+
+br_status finite_set_rewriter::mk_difference(expr * arg1, expr * arg2, expr_ref & result) {
+    // set.difference(x, x) -> set.empty
+    if (arg1 == arg2) {
+        // Get the set sort directly from the argument
+        sort* set_sort = arg1->get_sort();
+        SASSERT(m_util.is_finite_set(set_sort));
+        
+        // Call mk_empty with set_sort directly as suggested
+        result = m_util.mk_empty(set_sort);
+        return BR_DONE;
+    }
+    
+    return BR_FAILED;
+}
+
+br_status finite_set_rewriter::mk_subset(expr * arg1, expr * arg2, expr_ref & result) {
+    // set.subset(x, y) -> set.intersect(x, y) = x
+    expr_ref intersect(m());
+    intersect = m_util.mk_intersect(arg1, arg2);
+    result = m().mk_eq(intersect, arg1);
+    return BR_REWRITE3;
+}