mirrors/yosys
3
0
Fork 0
mirror of https://github.com/YosysHQ/yosys synced 2025-09-22 01:11:28 +00:00
Code Activity

Use a pool of ABC processes.

Browse source 
Doing ABC runs in parallel can actually make things slower when every ABC run requires
spawning an ABC subprocess --- especially when using popen(), which on glibc does not
use vfork(). What seems to happen is that constant fork()ing keeps making the main
process data pages copy-on-write, so the main process code that is setting up each ABC
call takes a lot of minor page-faults, slowing it down.

The solution is pretty straightforward although a little tricky to implement.
We just reuse ABC subprocesses. Instead of passing the ABC script name on the command
line, we spawn an ABC REPL and pipe a command into it to source the script. When that's
done we echo an `ABC_DONE` token instead of exiting. Yosys then puts the ABC process
onto a stack which we can pull from the next time we do an ABC run.

For one of our large designs, this is an additional 5x speedup of the primary AbcPass.
It does 5155 ABC runs, all very small; runtime of the AbcPass goes from 760s to 149s
(not very scientific benchmarking but the effect size is large).
This commit is contained in:
Robert O'Callahan 2025-08-08 05:26:30 +00:00 • committed by Jannis Harder
parent 27462da208
commit ae0ca7578a
2 changed files with 221 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

27
kernel/threading.h
View file

@ -154,6 +154,33 @@ private:
#endif
};
template <class T>
class ConcurrentStack
{
public:
void push_back(T &&t) {
#ifdef YOSYS_ENABLE_THREADS
std::lock_guard<std::mutex> lock(mutex);
#endif
contents.push_back(std::move(t));
}
std::optional<T> try_pop_back() {
#ifdef YOSYS_ENABLE_THREADS
std::lock_guard<std::mutex> lock(mutex);
#endif
if (contents.empty())
return std::nullopt;
T result = std::move(contents.back());
contents.pop_back();
return result;
}
private:
#ifdef YOSYS_ENABLE_THREADS
std::mutex mutex;
#endif
std::vector<T> contents;
};
YOSYS_NAMESPACE_END
#endif // YOSYS_THREADING_H

207
passes/techmap/abc.cc
View file

@ -59,6 +59,11 @@
#include <memory>
#include <vector>
#ifdef __linux__
# include <fcntl.h>
# include <spawn.h>
# include <sys/wait.h>
#endif
#ifndef _WIN32
# include <unistd.h>
# include <dirent.h>
@ -153,6 +158,121 @@ struct AbcSigVal {
}
};
#if defined(__linux__) && !defined(YOSYS_DISABLE_SPAWN)
struct AbcProcess
{
pid_t pid;
int to_child_pipe;
int from_child_pipe;
AbcProcess() : pid(0), to_child_pipe(-1), from_child_pipe(-1) {}
AbcProcess(AbcProcess &&other) {
pid = other.pid;
to_child_pipe = other.to_child_pipe;
from_child_pipe = other.from_child_pipe;
other.pid = 0;
other.to_child_pipe = other.from_child_pipe = -1;
}
AbcProcess &operator=(AbcProcess &&other) {
if (this != &other) {
pid = other.pid;
to_child_pipe = other.to_child_pipe;
from_child_pipe = other.from_child_pipe;
other.pid = 0;
other.to_child_pipe = other.from_child_pipe = -1;
}
return *this;
}
~AbcProcess() {
if (pid == 0)
return;
if (to_child_pipe >= 0)
close(to_child_pipe);
int status;
int ret = waitpid(pid, &status, 0);
if (ret != pid) {
log_error("waitpid(%d) failed", pid);
}
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
log_error("ABC failed with status %X", status);
}
if (from_child_pipe >= 0)
close(from_child_pipe);
}
};
std::optional<AbcProcess> spawn_abc(const char* abc_exe, DeferredLogs &logs) {
// Open pipes O_CLOEXEC so we don't leak any of the fds into racing
// fork()s.
int to_child_pipe[2];
if (pipe2(to_child_pipe, O_CLOEXEC) != 0) {
logs.log_error("pipe failed");
return std::nullopt;
}
int from_child_pipe[2];
if (pipe2(from_child_pipe, O_CLOEXEC) != 0) {
logs.log_error("pipe failed");
return std::nullopt;
}
AbcProcess result;
result.to_child_pipe = to_child_pipe[1];
result.from_child_pipe = from_child_pipe[0];
// Allow the child side of the pipes to be inherited.
fcntl(to_child_pipe[0], F_SETFD, 0);
fcntl(from_child_pipe[1], F_SETFD, 0);
posix_spawn_file_actions_t file_actions;
if (posix_spawn_file_actions_init(&file_actions) != 0) {
logs.log_error("posix_spawn_file_actions_init failed");
return std::nullopt;
}
if (posix_spawn_file_actions_addclose(&file_actions, to_child_pipe[1]) != 0) {
logs.log_error("posix_spawn_file_actions_addclose failed");
return std::nullopt;
}
if (posix_spawn_file_actions_addclose(&file_actions, from_child_pipe[0]) != 0) {
logs.log_error("posix_spawn_file_actions_addclose failed");
return std::nullopt;
}
if (posix_spawn_file_actions_adddup2(&file_actions, to_child_pipe[0], STDIN_FILENO) != 0) {
logs.log_error("posix_spawn_file_actions_adddup2 failed");
return std::nullopt;
}
if (posix_spawn_file_actions_adddup2(&file_actions, from_child_pipe[1], STDOUT_FILENO) != 0) {
logs.log_error("posix_spawn_file_actions_adddup2 failed");
return std::nullopt;
}
if (posix_spawn_file_actions_adddup2(&file_actions, from_child_pipe[1], STDERR_FILENO) != 0) {
logs.log_error("posix_spawn_file_actions_adddup2 failed");
return std::nullopt;
}
if (posix_spawn_file_actions_addclose(&file_actions, to_child_pipe[0]) != 0) {
logs.log_error("posix_spawn_file_actions_addclose failed");
return std::nullopt;
}
if (posix_spawn_file_actions_addclose(&file_actions, from_child_pipe[1]) != 0) {
logs.log_error("posix_spawn_file_actions_addclose failed");
return std::nullopt;
}
char arg1[] = "-s";
char* argv[] = { strdup(abc_exe), arg1, nullptr };
if (0 != posix_spawn(&result.pid, abc_exe, &file_actions, nullptr, argv, environ)) {
logs.log_error("posix_spawn %s failed", abc_exe);
return std::nullopt;
}
free(argv[0]);
posix_spawn_file_actions_destroy(&file_actions);
close(to_child_pipe[0]);
close(from_child_pipe[1]);
return result;
}
#else
struct AbcProcess {};
#endif
using AbcSigMap = SigValMap<AbcSigVal>;
// Used by off-main-threads. Contains no direct or indirect access to RTLIL.
@ -167,7 +287,7 @@ struct RunAbcState {
dict<int, std::string> pi_map, po_map;
RunAbcState(const AbcConfig &config) : config(config) {}
void run();
void run(ConcurrentStack<AbcProcess> &process_pool);
};
struct AbcModuleState {
@ -1010,7 +1130,42 @@ void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module
handle_loops(assign_map, module);
}
void RunAbcState::run()
bool read_until_abc_done(abc_output_filter &filt, int fd, DeferredLogs &logs) {
std::string line;
char buf[1024];
while (true) {
int ret = read(fd, buf, sizeof(buf) - 1);
if (ret < 0) {
logs.log_error("Failed to read from ABC, errno=%d", errno);
return false;
}
if (ret == 0) {
logs.log_error("ABC exited prematurely");
return false;
}
char *start = buf;
char *end = buf + ret;
while (start < end) {
char *p = static_cast<char*>(memchr(start, '\n', end - start));
if (p == nullptr) {
break;
}
line.append(start, p + 1 - start);
// ABC seems to actually print "ABC_DONE \n", but we probably shouldn't
// rely on that extra space being output.
if (line.substr(0, 8) == "ABC_DONE") {
// Ignore any leftover output, there should only be a prompt perhaps
return true;
}
filt.next_line(line);
line.clear();
start = p + 1;
}
line.append(start, end - start);
}
}
void RunAbcState::run(ConcurrentStack<AbcProcess> &process_pool)
{
std::string buffer = stringf("%s/input.blif", tempdir_name);
FILE *f = fopen(buffer.c_str(), "wt");
@ -1137,14 +1292,12 @@ void RunAbcState::run()
count_gates, GetSize(signal_list), count_input, count_output);
if (count_output > 0)
{
buffer = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file, tempdir_name);
logs.log("Running ABC command: %s\n", replace_tempdir(buffer, tempdir_name, config.show_tempdir));
std::string tmp_script_name = stringf("%s/abc.script", tempdir_name);
logs.log("Running ABC script: %s\n", replace_tempdir(tmp_script_name, tempdir_name, config.show_tempdir));
errno = 0;
#ifndef YOSYS_LINK_ABC
abc_output_filter filt(*this, tempdir_name, config.show_tempdir);
int ret = run_command(buffer, std::bind(&abc_output_filter::next_line, filt, std::placeholders::_1));
#else
#ifdef YOSYS_LINK_ABC
string temp_stdouterr_name = stringf("%s/stdouterr.txt", tempdir_name);
FILE *temp_stdouterr_w = fopen(temp_stdouterr_name.c_str(), "w");
if (temp_stdouterr_w == NULL)
@ -1165,7 +1318,6 @@ void RunAbcState::run()
fclose(temp_stdouterr_w);
// These needs to be mutable, supposedly due to getopt
char *abc_argv[5];
string tmp_script_name = stringf("%s/abc.script", tempdir_name);
abc_argv[0] = strdup(config.exe_file.c_str());
abc_argv[1] = strdup("-s");
abc_argv[2] = strdup("-f");
@ -1183,13 +1335,40 @@ void RunAbcState::run()
fclose(old_stdout);
fclose(old_stderr);
std::ifstream temp_stdouterr_r(temp_stdouterr_name);
abc_output_filter filt(*this, tempdir_name, config.show_tempdir);
for (std::string line; std::getline(temp_stdouterr_r, line); )
filt.next_line(line + "\n");
temp_stdouterr_r.close();
#elif defined(__linux__) && !defined(YOSYS_DISABLE_SPAWN)
AbcProcess process;
if (std::optional<AbcProcess> process_opt = process_pool.try_pop_back()) {
process = std::move(process_opt.value());
} else if (std::optional<AbcProcess> process_opt = spawn_abc(config.exe_file.c_str(), logs)) {
process = std::move(process_opt.value());
} else {
return;
}
std::string cmd = stringf(
// This makes ABC switch stdout to line buffering, which we need
// to see our ABC_DONE message.
"set abcout /dev/stdout\n"
"empty\n"
"source %s\n"
"echo \"ABC_DONE\"\n", tmp_script_name);
int ret = write(process.to_child_pipe, cmd.c_str(), cmd.size());
if (ret != static_cast<int>(cmd.size())) {
logs.log_error("write failed");
return;
}
ret = read_until_abc_done(filt, process.from_child_pipe, logs) ? 0 : 1;
if (ret == 0) {
process_pool.push_back(std::move(process));
}
#else
std::string cmd = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file.c_str(), tempdir_name.c_str());
int ret = run_command(cmd, std::bind(&abc_output_filter::next_line, filt, std::placeholders::_1));
#endif
if (ret != 0) {
logs.log_error("ABC: execution of command \"%s\" failed: return code %d (errno=%d).\n", buffer, ret, errno);
logs.log_error("ABC: execution of script \"%s\" failed: return code %d (errno=%d).\n", tmp_script_name, ret, errno);
return;
}
did_run = true;
@ -2205,7 +2384,8 @@ struct AbcPass : public Pass {
AbcModuleState state(config, initvals);
state.prepare_module(design, mod, assign_map, cells, dff_mode, clk_str);
state.run_abc.run();
ConcurrentStack<AbcProcess> process_pool;
state.run_abc.run(process_pool);
state.extract(assign_map, design, mod);
continue;
}
@ -2379,11 +2559,12 @@ struct AbcPass : public Pass {
ConcurrentQueue<std::unique_ptr<AbcModuleState>> work_queue(num_worker_threads);
ConcurrentQueue<std::unique_ptr<AbcModuleState>> work_finished_queue;
int work_finished_count = 0;
ConcurrentStack<AbcProcess> process_pool;
ThreadPool worker_threads(num_worker_threads, [&](int){
while (std::optional<std::unique_ptr<AbcModuleState>> work =
work_queue.pop_front()) {
// Only the `run_abc` component is safe to touch here!
(*work)->run_abc.run();
(*work)->run_abc.run(process_pool);
work_finished_queue.push_back(std::move(*work));
}
});
@ -2409,7 +2590,7 @@ struct AbcPass : public Pass {
work_queue.push_back(std::move(state));
} else {
// Just run everything on the main thread.
state->run_abc.run();
state->run_abc.run(process_pool);
work_finished_queue.push_back(std::move(state));
}
}