diff --git a/passes/opt/opt_dff.cc b/passes/opt/opt_dff.cc
index 8539432c0..4ed4b0cb6 100644
--- a/passes/opt/opt_dff.cc
+++ b/passes/opt/opt_dff.cc
@@ -738,7 +738,11 @@ struct OptDffWorker
 		ModWalker modwalker(module->design, module);
 		QuickConeSat qcsat(modwalker);
 
-		// Run as a separate sub-pass, so that we don't mutate (non-FF) cells under ModWalker.
+		// Defer mutating cells by removing them/emiting new flip flops so that
+		// cell references in modwalker are not invalidated
+		std::vector<RTLIL::Cell*> cells_to_remove;
+		std::vector<FfData> ffs_to_emit;
+
 		bool did_something = false;
 		for (auto cell : module->selected_cells()) {
 			if (!RTLIL::builtin_ff_cell_types().count(cell->type))
@@ -830,16 +834,20 @@ struct OptDffWorker
 					if (!removed_sigbits.count(i))
 						keep_bits.push_back(i);
 				if (keep_bits.empty()) {
-					module->remove(cell);
+					cells_to_remove.emplace_back(cell);
 					did_something = true;
 					continue;
 				}
 				ff = ff.slice(keep_bits);
 				ff.cell = cell;
-				ff.emit();
+				ffs_to_emit.emplace_back(ff);
 				did_something = true;
 			}
 		}
+		for (auto* cell : cells_to_remove)
+			module->remove(cell);
+		for (auto& ff : ffs_to_emit)
+			ff.emit();
 		return did_something;
 	}
 };
diff --git a/tests/opt/bug5164.ys b/tests/opt/bug5164.ys
new file mode 100644
index 000000000..4ee71fe45
--- /dev/null
+++ b/tests/opt/bug5164.ys
@@ -0,0 +1,60 @@
+read_rtlil <<EOT
+module \module137
+  wire input 1 \clk
+  wire width 1 output 1 \qa
+  wire width 1 \qb
+  cell $dff \dffa
+    parameter \CLK_POLARITY 1
+    parameter \WIDTH 1
+    connect \CLK \clk
+    connect \D \qb
+    connect \Q \qa
+  end
+  cell $dff \dffb
+    parameter \CLK_POLARITY 1
+    parameter \WIDTH 1
+    connect \CLK \clk
+    connect \D 1'x
+    connect \Q \qb
+  end
+end
+EOT
+equiv_opt -assert opt_dff -sat
+
+design -reset
+
+read_rtlil <<EOT
+module \module137
+  wire output 1 width 9 $2\reg204[8:0]
+  wire input 1 \clk
+  wire width 9 $auto$wreduce.cc:514:run$19340
+  wire width 9 $auto$wreduce.cc:514:run$19341
+  wire width 15 \dffout
+  attribute \init 9'000000000
+  wire width 9 \reg204
+  cell $dff $auto$ff.cc:266:slice$26225
+    parameter \CLK_POLARITY 1
+    parameter \WIDTH 15
+    connect \CLK \clk
+    connect \D { 9'x \reg204 [8:3] }
+    connect \Q \dffout
+  end
+  cell $dff $auto$ff.cc:266:slice$26292
+    parameter \CLK_POLARITY 1
+    parameter \WIDTH 9
+    connect \CLK \clk
+    connect \D $2\reg204[8:0]
+    connect \Q \reg204
+  end
+  cell $mux $procmux$4510
+    parameter \WIDTH 9
+    connect \A 9'x
+    connect \B 9'x
+    connect \S 1'x
+    connect \Y $auto$wreduce.cc:514:run$19340
+  end
+  connect $2\reg204[8:0] $auto$wreduce.cc:514:run$19340
+  connect $auto$wreduce.cc:514:run$19341 [8:3] 6'000000
+end
+EOT
+equiv_opt -assert opt_dff -sat