Merge pull request #4834 from YosysHQ/emil/gzip-refactor

Memory-efficient zlib usage across Liberty file consumers

Makefile

@@ -586,7 +586,9 @@ $(eval $(call add_include_file,kernel/fmt.h))
 ifeq ($(ENABLE_ZLIB),1)
 $(eval $(call add_include_file,kernel/fstdata.h))
 endif
+$(eval $(call add_include_file,kernel/gzip.h))
 $(eval $(call add_include_file,kernel/hashlib.h))
+$(eval $(call add_include_file,kernel/io.h))
 $(eval $(call add_include_file,kernel/json.h))
 $(eval $(call add_include_file,kernel/log.h))
 $(eval $(call add_include_file,kernel/macc.h))
@@ -617,7 +619,7 @@ $(eval $(call add_include_file,frontends/ast/ast_binding.h))
 $(eval $(call add_include_file,frontends/blif/blifparse.h))
 $(eval $(call add_include_file,backends/rtlil/rtlil_backend.h))
 
-OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o
+OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o kernel/io.o kernel/gzip.o
 OBJS += kernel/binding.o kernel/tclapi.o
 OBJS += kernel/cellaigs.o kernel/celledges.o kernel/cost.o kernel/satgen.o kernel/scopeinfo.o kernel/qcsat.o kernel/mem.o kernel/ffmerge.o kernel/ff.o kernel/yw.o kernel/json.o kernel/fmt.o kernel/sexpr.o
 OBJS += kernel/drivertools.o kernel/functional.o

backends/btor/btor.cc

@@ -30,6 +30,7 @@
 #include "kernel/mem.h"
 #include "kernel/json.h"
 #include "kernel/yw.h"
+#include "kernel/utils.h"
 #include <string>
 
 USING_YOSYS_NAMESPACE

backends/smt2/smt2.cc

@@ -24,6 +24,7 @@
 #include "kernel/log.h"
 #include "kernel/mem.h"
 #include "libs/json11/json11.hpp"
+#include "kernel/utils.h"
 #include <string>
 
 USING_YOSYS_NAMESPACE

frontends/verific/verific.cc

@@ -21,6 +21,7 @@
 #include "kernel/sigtools.h"
 #include "kernel/celltypes.h"
 #include "kernel/log.h"
+#include "kernel/utils.h"
 #include "libs/sha1/sha1.h"
 #include <stdlib.h>
 #include <stdio.h>

kernel/gzip.cc

@@ -0,0 +1,139 @@
+#include "kernel/yosys_common.h"
+#include "kernel/log.h"
+#include "kernel/gzip.h"
+#include <iostream>
+#include <string>
+#include <cstdarg>
+#include <cstdio>
+
+#if !defined(WIN32)
+#include <dirent.h>
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+
+YOSYS_NAMESPACE_BEGIN
+
+#ifdef YOSYS_ENABLE_ZLIB
+
+gzip_ostream::obuf::obuf() {
+    setp(buffer, buffer + buffer_size - 1);
+}
+
+bool gzip_ostream::obuf::open(const std::string &filename) {
+    gzf = Zlib::gzopen(filename.c_str(), "wb");
+    return gzf != nullptr;
+}
+
+int gzip_ostream::obuf::sync() {
+    int num = pptr() - pbase();
+    if (num > 0) {
+        if (Zlib::gzwrite(gzf, reinterpret_cast<const void*>(pbase()), num) != num) {
+            return -1;
+        }
+        pbump(-num);
+    }
+    return 0;
+}
+
+gzip_ostream::obuf::~obuf() {
+    if (gzf) {
+        sync();
+        Zlib::gzclose(gzf);
+    }
+}
+
+bool gzip_istream::ibuf::open(const std::string& filename) {
+	if (gzf) {
+		Zlib::gzclose(gzf);
+	}
+	gzf = Zlib::gzopen(filename.c_str(), "rb");
+	if (!gzf) {
+		return false;
+	}
+	// Empty and point to start
+	setg(buffer, buffer, buffer);
+	return true;
+}
+
+// Called when the buffer is empty and more input is needed
+std::istream::int_type gzip_istream::ibuf::underflow() {
+	log_assert(gzf && "No gzfile opened\n");
+	int bytes_read = Zlib::gzread(gzf, buffer, buffer_size);
+	if (bytes_read <= 0) {
+		if (Zlib::gzeof(gzf)) {
+			// "On failure, the function ensures that either
+			// gptr() == nullptr or gptr() == egptr."
+			// Let's set gptr to egptr
+			setg(eback(), egptr(), egptr());
+			return traits_type::eof();
+		}
+
+		int err;
+		const char* error_msg = Zlib::gzerror(gzf, &err);
+		if (err != Z_OK)
+			log_error("%s", error_msg);
+		else
+			log_error("Decompression logic failure: "\
+					"read <=0 bytes but neither EOF nor error\n");
+	}
+
+	// Keep size and point to start
+	setg(buffer, buffer, buffer + bytes_read);
+	return traits_type::to_int_type(buffer[0]);
+}
+
+gzip_istream::ibuf::~ibuf() {
+	if (gzf) {
+		int err = Zlib::gzclose(gzf);
+		if (err != Z_OK) {
+			// OK to overwrite rr it, it doesn't change
+			const char* error_msg = Zlib::gzerror(gzf, &err);
+			log_error("%s", error_msg);
+		}
+	}
+}
+
+#endif // YOSYS_ENABLE_ZLIB
+
+
+// Takes a successfully opened ifstream. If it's gzipped, returns an istream. Otherwise,
+// returns the original ifstream, rewound to the start.
+std::istream* uncompressed(const std::string filename, std::ios_base::openmode mode) {
+	std::ifstream* f = new std::ifstream();
+	f->open(filename, mode);
+	if (f->fail())
+		return f;
+	// Check for gzip magic
+	unsigned char magic[3];
+	int n = 0;
+	while (n < 3)
+	{
+		int c = f->get();
+		if (c != EOF) {
+			magic[n] = (unsigned char) c;
+		}
+		n++;
+	}
+	if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) {
+#ifdef YOSYS_ENABLE_ZLIB
+		log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str());
+		if (magic[2] != 8)
+			log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n",
+				filename.c_str(), unsigned(magic[2]));
+		gzip_istream* s = new gzip_istream();
+		delete f;
+		s->open(filename.c_str());
+		return s;
+#else
+		log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str());
+#endif // YOSYS_ENABLE_ZLIB
+	} else {
+		f->clear();
+		f->seekg(0, std::ios::beg);
+		return f;
+	}
+}
+
+YOSYS_NAMESPACE_END

kernel/gzip.h

@@ -0,0 +1,78 @@
+#include <string>
+#include "kernel/yosys_common.h"
+
+#ifndef YOSYS_GZIP_H
+#define YOSYS_GZIP_H
+
+YOSYS_NAMESPACE_BEGIN
+
+#ifdef YOSYS_ENABLE_ZLIB
+
+namespace Zlib {
+#include <zlib.h>
+}
+
+/*
+An output stream that uses a stringbuf to buffer data internally,
+using zlib to write gzip-compressed data every time the stream is flushed.
+*/
+class gzip_ostream : public std::ostream {
+public:
+	gzip_ostream(): std::ostream(nullptr) {
+		rdbuf(&outbuf);
+	}
+	bool open(const std::string &filename) {
+		return outbuf.open(filename);
+	}
+private:
+	class obuf : public std::stringbuf {
+	public:
+		obuf();
+		bool open(const std::string &filename);
+		virtual int sync() override;
+		virtual ~obuf();
+	private:
+		static const int buffer_size = 4096;
+		char buffer[buffer_size];             // Internal buffer for compressed data
+		Zlib::gzFile gzf = nullptr;                 // Handle to the gzip file
+	};
+
+	obuf outbuf;  // The stream buffer instance
+};
+
+/*
+An input stream that uses zlib to read gzip-compressed data from a file,
+buffering the decompressed data internally using its own buffer.
+*/
+class gzip_istream final : public std::istream {
+public:
+	gzip_istream() : std::istream(&inbuf) {}
+	bool open(const std::string& filename) {
+		return inbuf.open(filename);
+	}
+private:
+	class ibuf final : public std::streambuf {
+	public:
+		ibuf() : gzf(nullptr) {}
+		bool open(const std::string& filename);
+		virtual ~ibuf();
+
+	protected:
+		// Called when the buffer is empty and more input is needed
+		virtual int_type underflow() override;
+	private:
+		static const int buffer_size = 8192;
+		char buffer[buffer_size];
+		Zlib::gzFile gzf;
+	};
+
+	ibuf inbuf;  // The stream buffer instance
+};
+
+#endif // YOSYS_ENABLE_ZLIB
+
+std::istream* uncompressed(const std::string filename, std::ios_base::openmode mode = std::ios_base::in);
+
+YOSYS_NAMESPACE_END
+
+#endif // YOSYS_GZIP_H

kernel/io.cc

@@ -0,0 +1,370 @@
+#include "kernel/yosys_common.h"
+#include "kernel/log.h"
+#include <iostream>
+#include <string>
+
+#if !defined(WIN32)
+#include <dirent.h>
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+
+YOSYS_NAMESPACE_BEGIN
+
+// Set of utilities for handling files
+
+int readsome(std::istream &f, char *s, int n)
+{
+	int rc = int(f.readsome(s, n));
+
+	// f.readsome() sometimes returns 0 on a non-empty stream..
+	if (rc == 0) {
+		int c = f.get();
+		if (c != EOF) {
+			*s = c;
+			rc = 1;
+		}
+	}
+
+	return rc;
+}
+
+std::string next_token(std::string &text, const char *sep, bool long_strings)
+{
+	size_t pos_begin = text.find_first_not_of(sep);
+
+	if (pos_begin == std::string::npos)
+		pos_begin = text.size();
+
+	if (long_strings && pos_begin != text.size() && text[pos_begin] == '"') {
+		std::string sep_string = sep;
+		for (size_t i = pos_begin+1; i < text.size(); i++) {
+			if (text[i] == '"' && (i+1 == text.size() || sep_string.find(text[i+1]) != std::string::npos)) {
+				std::string token = text.substr(pos_begin, i-pos_begin+1);
+				text = text.substr(i+1);
+				return token;
+			}
+			if (i+1 < text.size() && text[i] == '"' && text[i+1] == ';' && (i+2 == text.size() || sep_string.find(text[i+2]) != std::string::npos)) {
+				std::string token = text.substr(pos_begin, i-pos_begin+1);
+				text = text.substr(i+2);
+				return token + ";";
+			}
+		}
+	}
+
+	size_t pos_end = text.find_first_of(sep, pos_begin);
+
+	if (pos_end == std::string::npos)
+		pos_end = text.size();
+
+	std::string token = text.substr(pos_begin, pos_end-pos_begin);
+	text = text.substr(pos_end);
+	return token;
+}
+
+std::vector<std::string> split_tokens(const std::string &text, const char *sep)
+{
+	std::vector<std::string> tokens;
+	std::string current_token;
+	for (char c : text) {
+		if (strchr(sep, c)) {
+			if (!current_token.empty()) {
+				tokens.push_back(current_token);
+				current_token.clear();
+			}
+		} else
+			current_token += c;
+	}
+	if (!current_token.empty()) {
+		tokens.push_back(current_token);
+		current_token.clear();
+	}
+	return tokens;
+}
+
+// this is very similar to fnmatch(). the exact rules used by this
+// function are:
+//
+//    ?        matches any character except
+//    *        matches any sequence of characters
+//    [...]    matches any of the characters in the list
+//    [!..]    matches any of the characters not in the list
+//
+// a backslash may be used to escape the next characters in the
+// pattern. each special character can also simply match itself.
+//
+bool patmatch(const char *pattern, const char *string)
+{
+	if (*pattern == 0)
+		return *string == 0;
+
+	if (*pattern == '\\') {
+		if (pattern[1] == string[0] && patmatch(pattern+2, string+1))
+			return true;
+	}
+
+	if (*pattern == '?') {
+		if (*string == 0)
+			return false;
+		return patmatch(pattern+1, string+1);
+	}
+
+	if (*pattern == '*') {
+		while (*string) {
+			if (patmatch(pattern+1, string++))
+				return true;
+		}
+		return pattern[1] == 0;
+	}
+
+	if (*pattern == '[') {
+		bool found_match = false;
+		bool inverted_list = pattern[1] == '!';
+		const char *p = pattern + (inverted_list ? 1 : 0);
+
+		while (*++p) {
+			if (*p == ']') {
+				if (found_match != inverted_list && patmatch(p+1, string+1))
+					return true;
+				break;
+			}
+
+			if (*p == '\\') {
+				if (*++p == *string)
+					found_match = true;
+			} else
+			if (*p == *string)
+				found_match = true;
+		}
+	}
+
+	if (*pattern == *string)
+		return patmatch(pattern+1, string+1);
+
+	return false;
+}
+
+std::string get_base_tmpdir()
+{
+	static std::string tmpdir;
+
+	if (!tmpdir.empty()) {
+		return tmpdir;
+	}
+
+#if defined(_WIN32)
+#  ifdef __MINGW32__
+	char longpath[MAX_PATH + 1];
+	char shortpath[MAX_PATH + 1];
+#  else
+	WCHAR longpath[MAX_PATH + 1];
+	TCHAR shortpath[MAX_PATH + 1];
+#  endif
+	if (!GetTempPath(MAX_PATH+1, longpath))
+		log_error("GetTempPath() failed.\n");
+	if (!GetShortPathName(longpath, shortpath, MAX_PATH + 1))
+		log_error("GetShortPathName() failed.\n");
+	for (int i = 0; shortpath[i]; i++)
+		tmpdir += char(shortpath[i]);
+#else
+	char * var = std::getenv("TMPDIR");
+	if (var && strlen(var)!=0) {
+		tmpdir.assign(var);
+		// We return the directory name without the trailing '/'
+		while (!tmpdir.empty() && (tmpdir.back() == '/')) {
+			tmpdir.pop_back();
+		}
+	} else {
+		tmpdir.assign("/tmp");
+	}
+#endif
+	return tmpdir;
+}
+
+std::string make_temp_file(std::string template_str)
+{
+	size_t pos = template_str.rfind("XXXXXX");
+	log_assert(pos != std::string::npos);
+#if defined(__wasm)
+	static size_t index = 0;
+	template_str.replace(pos, 6, stringf("%06zu", index++));
+#elif defined(_WIN32)
+#ifndef YOSYS_WIN32_UNIX_DIR
+	std::replace(template_str.begin(), template_str.end(), '/', '\\');
+#endif
+	while (1) {
+		for (int i = 0; i < 6; i++) {
+			static std::string y = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+			static uint32_t x = 314159265 ^ uint32_t(time(NULL));
+			x ^= x << 13, x ^= x >> 17, x ^= x << 5;
+			template_str[pos+i] = y[x % y.size()];
+		}
+		if (_access(template_str.c_str(), 0) != 0)
+			break;
+	}
+#else
+	int suffixlen = template_str.size() - pos - 6;
+
+	char *p = strdup(template_str.c_str());
+	close(mkstemps(p, suffixlen));
+	template_str = p;
+	free(p);
+#endif
+
+	return template_str;
+}
+
+std::string make_temp_dir(std::string template_str)
+{
+#if defined(_WIN32)
+	template_str = make_temp_file(template_str);
+	mkdir(template_str.c_str());
+	return template_str;
+#elif defined(__wasm)
+	template_str = make_temp_file(template_str);
+	mkdir(template_str.c_str(), 0777);
+	return template_str;
+#else
+#  ifndef NDEBUG
+	size_t pos = template_str.rfind("XXXXXX");
+	log_assert(pos != std::string::npos);
+
+	int suffixlen = template_str.size() - pos - 6;
+	log_assert(suffixlen == 0);
+#  endif
+
+	char *p = strdup(template_str.c_str());
+	log_assert(p);
+	char *res = mkdtemp(p);
+	if (!res)
+		log_error("mkdtemp failed for '%s': %s [Error %d]\n",
+			p, strerror(errno), errno);
+	template_str = p;
+	free(p);
+
+	return template_str;
+#endif
+}
+
+bool check_directory_exists(const std::string& dirname)
+{
+#if defined(_WIN32)
+	struct _stat info;
+	if (_stat(dirname.c_str(), &info) != 0)
+	{
+		return false;
+	}
+	return (info.st_mode & _S_IFDIR) != 0;
+#else
+	struct stat info;
+	if (stat(dirname.c_str(), &info) != 0)
+	{
+		return false;
+	}
+	return (info.st_mode & S_IFDIR) != 0;
+#endif
+}
+
+#ifdef _WIN32
+bool check_file_exists(std::string filename, bool)
+{
+	return _access(filename.c_str(), 0) == 0;
+}
+#else
+bool check_file_exists(std::string filename, bool is_exec)
+{
+	return access(filename.c_str(), is_exec ? X_OK : F_OK) == 0;
+}
+#endif
+
+bool is_absolute_path(std::string filename)
+{
+#ifdef _WIN32
+	return filename[0] == '/' || filename[0] == '\\' || (filename[0] != 0 && filename[1] == ':');
+#else
+	return filename[0] == '/';
+#endif
+}
+
+void remove_directory(std::string dirname)
+{
+#ifdef _WIN32
+	run_command(stringf("rmdir /s /q \"%s\"", dirname.c_str()));
+#else
+	struct stat stbuf;
+	struct dirent **namelist;
+	int n = scandir(dirname.c_str(), &namelist, nullptr, alphasort);
+	log_assert(n >= 0);
+	for (int i = 0; i < n; i++) {
+		if (strcmp(namelist[i]->d_name, ".") && strcmp(namelist[i]->d_name, "..")) {
+			std::string buffer = stringf("%s/%s", dirname.c_str(), namelist[i]->d_name);
+			if (!stat(buffer.c_str(), &stbuf) && S_ISREG(stbuf.st_mode)) {
+				remove(buffer.c_str());
+			} else
+				remove_directory(buffer);
+		}
+		free(namelist[i]);
+	}
+	free(namelist);
+	rmdir(dirname.c_str());
+#endif
+}
+
+bool create_directory(const std::string& dirname)
+{
+#if defined(_WIN32)
+	int ret = _mkdir(dirname.c_str());
+#else
+	mode_t mode = 0755;
+	int ret = mkdir(dirname.c_str(), mode);
+#endif
+	if (ret == 0)
+		return true;
+
+	switch (errno)
+	{
+	case ENOENT:
+		// parent didn't exist, try to create it
+		{
+			std::string::size_type pos = dirname.find_last_of('/');
+			if (pos == std::string::npos)
+#if defined(_WIN32)
+				pos = dirname.find_last_of('\\');
+			if (pos == std::string::npos)
+#endif
+				return false;
+			if (!create_directory( dirname.substr(0, pos) ))
+				return false;
+		}
+		// now, try to create again
+#if defined(_WIN32)
+		return 0 == _mkdir(dirname.c_str());
+#else
+		return 0 == mkdir(dirname.c_str(), mode);
+#endif
+
+	case EEXIST:
+		// done!
+		return check_directory_exists(dirname);
+
+	default:
+		return false;
+	}
+}
+
+std::string escape_filename_spaces(const std::string& filename)
+{
+	std::string out;
+	out.reserve(filename.size());
+	for (auto c : filename)
+	{
+		if (c == ' ')
+			out += "\\ ";
+		else
+			out.push_back(c);
+	}
+	return out;
+}
+
+YOSYS_NAMESPACE_END

kernel/io.h

@@ -0,0 +1,69 @@
+#include <string>
+#include <stdarg.h>
+#include "kernel/yosys_common.h"
+
+#ifndef YOSYS_IO_H
+#define YOSYS_IO_H
+
+YOSYS_NAMESPACE_BEGIN
+
+inline std::string vstringf(const char *fmt, va_list ap)
+{
+	// For the common case of strings shorter than 128, save a heap
+	// allocation by using a stack allocated buffer.
+	const int kBufSize = 128;
+	char buf[kBufSize];
+	buf[0] = '\0';
+	va_list apc;
+	va_copy(apc, ap);
+	int n = vsnprintf(buf, kBufSize, fmt, apc);
+	va_end(apc);
+	if (n < kBufSize)
+		return std::string(buf);
+
+	std::string string;
+	char *str = NULL;
+#if defined(_WIN32) || defined(__CYGWIN__)
+	int sz = 2 * kBufSize, rc;
+	while (1) {
+		va_copy(apc, ap);
+		str = (char *)realloc(str, sz);
+		rc = vsnprintf(str, sz, fmt, apc);
+		va_end(apc);
+		if (rc >= 0 && rc < sz)
+			break;
+		sz *= 2;
+	}
+	if (str != NULL) {
+		string = str;
+		free(str);
+	}
+	return string;
+#else
+	if (vasprintf(&str, fmt, ap) < 0)
+		str = NULL;
+	if (str != NULL) {
+		string = str;
+		free(str);
+	}
+	return string;
+#endif
+}
+
+std::string stringf(const char *fmt, ...) YS_ATTRIBUTE(format(printf, 1, 2));
+
+inline std::string stringf(const char *fmt, ...)
+{
+	std::string string;
+	va_list ap;
+
+	va_start(ap, fmt);
+	string = vstringf(fmt, ap);
+	va_end(ap);
+
+	return string;
+}
+
+YOSYS_NAMESPACE_END
+
+#endif // YOSYS_IO_H

kernel/register.cc

@@ -20,71 +20,13 @@
 #include "kernel/yosys.h"
 #include "kernel/satgen.h"
 #include "kernel/json.h"
+#include "kernel/gzip.h"
 
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
 
-#ifdef YOSYS_ENABLE_ZLIB
-#include <zlib.h>
-
-PRIVATE_NAMESPACE_BEGIN
-#define GZ_BUFFER_SIZE 8192
-void decompress_gzip(const std::string &filename, std::stringstream &out)
-{
-	char buffer[GZ_BUFFER_SIZE];
-	int bytes_read;
-	gzFile gzf = gzopen(filename.c_str(), "rb");
-	while(!gzeof(gzf)) {
-		bytes_read = gzread(gzf, reinterpret_cast<void *>(buffer), GZ_BUFFER_SIZE);
-		out.write(buffer, bytes_read);
-	}
-	gzclose(gzf);
-}
-
-/*
-An output stream that uses a stringbuf to buffer data internally,
-using zlib to write gzip-compressed data every time the stream is flushed.
-*/
-class gzip_ostream : public std::ostream  {
-public:
-	gzip_ostream() : std::ostream(nullptr)
-	{
-		rdbuf(&outbuf);
-	}
-	bool open(const std::string &filename)
-	{
-		return outbuf.open(filename);
-	}
-private:
-	class gzip_streambuf : public std::stringbuf {
-	public:
-		gzip_streambuf() { };
-		bool open(const std::string &filename)
-		{
-			gzf = gzopen(filename.c_str(), "wb");
-			return gzf != nullptr;
-		}
-		virtual int sync() override
-		{
-			gzwrite(gzf, reinterpret_cast<const void *>(str().c_str()), unsigned(str().size()));
-			str("");
-			return 0;
-		}
-		virtual ~gzip_streambuf()
-		{
-			sync();
-			gzclose(gzf);
-		}
-	private:
-		gzFile gzf = nullptr;
-	} outbuf;
-};
-PRIVATE_NAMESPACE_END
-
-#endif
-
 YOSYS_NAMESPACE_BEGIN
 
 #define MAX_REG_COUNT 1000
@@ -527,44 +469,8 @@ void Frontend::extra_args(std::istream *&f, std::string &filename, std::vector<s
 				next_args.insert(next_args.end(), args.begin(), args.begin()+argidx);
 				next_args.insert(next_args.end(), filenames.begin()+1, filenames.end());
 			}
-			std::ifstream *ff = new std::ifstream;
-			ff->open(filename.c_str(), bin_input ? std::ifstream::binary : std::ifstream::in);
 			yosys_input_files.insert(filename);
-			if (ff->fail()) {
-				delete ff;
-				ff = nullptr;
-			}
-			f = ff;
-			if (f != NULL) {
-				// Check for gzip magic
-				unsigned char magic[3];
-				int n = 0;
-				while (n < 3)
-				{
-					int c = ff->get();
-					if (c != EOF) {
-						magic[n] = (unsigned char) c;
-					}
-					n++;
-				}
-				if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) {
-	#ifdef YOSYS_ENABLE_ZLIB
-					log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str());
-					if (magic[2] != 8)
-						log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n",
-							filename.c_str(), unsigned(magic[2]));
-					delete ff;
-					std::stringstream *df = new std::stringstream();
-					decompress_gzip(filename, *df);
-					f = df;
-	#else
-					log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str());
-	#endif
-				} else {
-					ff->clear();
-					ff->seekg(0, std::ios::beg);
-				}
-			}
+			f = uncompressed(filename, bin_input ? std::ifstream::binary : std::ifstream::in);
 		}
 		if (f == NULL)
 			log_cmd_error("Can't open input file `%s' for reading: %s\n", filename.c_str(), strerror(errno));

kernel/utils.h

@@ -262,6 +262,20 @@ struct arrow_proxy {
 	T* operator->() { return &v; }
 };
 
+inline int ceil_log2(int x)
+{
+#if defined(__GNUC__)
+        return x > 1 ? (8*sizeof(int)) - __builtin_clz(x-1) : 0;
+#else
+	if (x <= 0)
+		return 0;
+	for (int i = 0; i < 32; i++)
+		if (((x-1) >> i) == 0)
+			return i;
+	log_abort();
+#endif
+}
+
 YOSYS_NAMESPACE_END
 
 #endif

kernel/yosys.cc

@@ -147,151 +147,6 @@ void yosys_banner()
 	log(" %s\n", yosys_version_str);
 }
 
-int ceil_log2(int x)
-{
-#if defined(__GNUC__)
-        return x > 1 ? (8*sizeof(int)) - __builtin_clz(x-1) : 0;
-#else
-	if (x <= 0)
-		return 0;
-	for (int i = 0; i < 32; i++)
-		if (((x-1) >> i) == 0)
-			return i;
-	log_abort();
-#endif
-}
-
-int readsome(std::istream &f, char *s, int n)
-{
-	int rc = int(f.readsome(s, n));
-
-	// f.readsome() sometimes returns 0 on a non-empty stream..
-	if (rc == 0) {
-		int c = f.get();
-		if (c != EOF) {
-			*s = c;
-			rc = 1;
-		}
-	}
-
-	return rc;
-}
-
-std::string next_token(std::string &text, const char *sep, bool long_strings)
-{
-	size_t pos_begin = text.find_first_not_of(sep);
-
-	if (pos_begin == std::string::npos)
-		pos_begin = text.size();
-
-	if (long_strings && pos_begin != text.size() && text[pos_begin] == '"') {
-		string sep_string = sep;
-		for (size_t i = pos_begin+1; i < text.size(); i++) {
-			if (text[i] == '"' && (i+1 == text.size() || sep_string.find(text[i+1]) != std::string::npos)) {
-				std::string token = text.substr(pos_begin, i-pos_begin+1);
-				text = text.substr(i+1);
-				return token;
-			}
-			if (i+1 < text.size() && text[i] == '"' && text[i+1] == ';' && (i+2 == text.size() || sep_string.find(text[i+2]) != std::string::npos)) {
-				std::string token = text.substr(pos_begin, i-pos_begin+1);
-				text = text.substr(i+2);
-				return token + ";";
-			}
-		}
-	}
-
-	size_t pos_end = text.find_first_of(sep, pos_begin);
-
-	if (pos_end == std::string::npos)
-		pos_end = text.size();
-
-	std::string token = text.substr(pos_begin, pos_end-pos_begin);
-	text = text.substr(pos_end);
-	return token;
-}
-
-std::vector<std::string> split_tokens(const std::string &text, const char *sep)
-{
-	std::vector<std::string> tokens;
-	std::string current_token;
-	for (char c : text) {
-		if (strchr(sep, c)) {
-			if (!current_token.empty()) {
-				tokens.push_back(current_token);
-				current_token.clear();
-			}
-		} else
-			current_token += c;
-	}
-	if (!current_token.empty()) {
-		tokens.push_back(current_token);
-		current_token.clear();
-	}
-	return tokens;
-}
-
-// this is very similar to fnmatch(). the exact rules used by this
-// function are:
-//
-//    ?        matches any character except
-//    *        matches any sequence of characters
-//    [...]    matches any of the characters in the list
-//    [!..]    matches any of the characters not in the list
-//
-// a backslash may be used to escape the next characters in the
-// pattern. each special character can also simply match itself.
-//
-bool patmatch(const char *pattern, const char *string)
-{
-	if (*pattern == 0)
-		return *string == 0;
-
-	if (*pattern == '\\') {
-		if (pattern[1] == string[0] && patmatch(pattern+2, string+1))
-			return true;
-	}
-
-	if (*pattern == '?') {
-		if (*string == 0)
-			return false;
-		return patmatch(pattern+1, string+1);
-	}
-
-	if (*pattern == '*') {
-		while (*string) {
-			if (patmatch(pattern+1, string++))
-				return true;
-		}
-		return pattern[1] == 0;
-	}
-
-	if (*pattern == '[') {
-		bool found_match = false;
-		bool inverted_list = pattern[1] == '!';
-		const char *p = pattern + (inverted_list ? 1 : 0);
-
-		while (*++p) {
-			if (*p == ']') {
-				if (found_match != inverted_list && patmatch(p+1, string+1))
-					return true;
-				break;
-			}
-
-			if (*p == '\\') {
-				if (*++p == *string)
-					found_match = true;
-			} else
-			if (*p == *string)
-				found_match = true;
-		}
-	}
-
-	if (*pattern == *string)
-		return patmatch(pattern+1, string+1);
-
-	return false;
-}
-
 #if !defined(YOSYS_DISABLE_SPAWN)
 int run_command(const std::string &command, std::function<void(const std::string&)> process_line)
 {
@@ -323,228 +178,6 @@ int run_command(const std::string &command, std::function<void(const std::string
 }
 #endif
 
-std::string get_base_tmpdir()
-{
-	static std::string tmpdir;
-
-	if (!tmpdir.empty()) {
-		return tmpdir;
-	}
-
-#if defined(_WIN32)
-#  ifdef __MINGW32__
-	char longpath[MAX_PATH + 1];
-	char shortpath[MAX_PATH + 1];
-#  else
-	WCHAR longpath[MAX_PATH + 1];
-	TCHAR shortpath[MAX_PATH + 1];
-#  endif
-	if (!GetTempPath(MAX_PATH+1, longpath))
-		log_error("GetTempPath() failed.\n");
-	if (!GetShortPathName(longpath, shortpath, MAX_PATH + 1))
-		log_error("GetShortPathName() failed.\n");
-	for (int i = 0; shortpath[i]; i++)
-		tmpdir += char(shortpath[i]);
-#else
-	char * var = std::getenv("TMPDIR");
-	if (var && strlen(var)!=0) {
-		tmpdir.assign(var);
-		// We return the directory name without the trailing '/'
-		while (!tmpdir.empty() && (tmpdir.back() == '/')) {
-			tmpdir.pop_back();
-		}
-	} else {
-		tmpdir.assign("/tmp");
-	}
-#endif
-	return tmpdir;
-}
-
-std::string make_temp_file(std::string template_str)
-{
-	size_t pos = template_str.rfind("XXXXXX");
-	log_assert(pos != std::string::npos);
-#if defined(__wasm)
-	static size_t index = 0;
-	template_str.replace(pos, 6, stringf("%06zu", index++));
-#elif defined(_WIN32)
-#ifndef YOSYS_WIN32_UNIX_DIR
-	std::replace(template_str.begin(), template_str.end(), '/', '\\');
-#endif
-	while (1) {
-		for (int i = 0; i < 6; i++) {
-			static std::string y = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
-			static uint32_t x = 314159265 ^ uint32_t(time(NULL));
-			x ^= x << 13, x ^= x >> 17, x ^= x << 5;
-			template_str[pos+i] = y[x % y.size()];
-		}
-		if (_access(template_str.c_str(), 0) != 0)
-			break;
-	}
-#else
-	int suffixlen = GetSize(template_str) - pos - 6;
-
-	char *p = strdup(template_str.c_str());
-	close(mkstemps(p, suffixlen));
-	template_str = p;
-	free(p);
-#endif
-
-	return template_str;
-}
-
-std::string make_temp_dir(std::string template_str)
-{
-#if defined(_WIN32)
-	template_str = make_temp_file(template_str);
-	mkdir(template_str.c_str());
-	return template_str;
-#elif defined(__wasm)
-	template_str = make_temp_file(template_str);
-	mkdir(template_str.c_str(), 0777);
-	return template_str;
-#else
-#  ifndef NDEBUG
-	size_t pos = template_str.rfind("XXXXXX");
-	log_assert(pos != std::string::npos);
-
-	int suffixlen = GetSize(template_str) - pos - 6;
-	log_assert(suffixlen == 0);
-#  endif
-
-	char *p = strdup(template_str.c_str());
-	log_assert(p);
-	char *res = mkdtemp(p);
-	if (!res)
-		log_error("mkdtemp failed for '%s': %s [Error %d]\n",
-			p, strerror(errno), errno);
-	template_str = p;
-	free(p);
-
-	return template_str;
-#endif
-}
-
-bool check_directory_exists(const std::string& dirname)
-{
-#if defined(_WIN32)
-	struct _stat info;
-	if (_stat(dirname.c_str(), &info) != 0)
-	{
-		return false;
-	}
-	return (info.st_mode & _S_IFDIR) != 0;
-#else
-	struct stat info;
-	if (stat(dirname.c_str(), &info) != 0)
-	{
-		return false;
-	}
-	return (info.st_mode & S_IFDIR) != 0;
-#endif
-}
-
-#ifdef _WIN32
-bool check_file_exists(std::string filename, bool)
-{
-	return _access(filename.c_str(), 0) == 0;
-}
-#else
-bool check_file_exists(std::string filename, bool is_exec)
-{
-	return access(filename.c_str(), is_exec ? X_OK : F_OK) == 0;
-}
-#endif
-
-bool is_absolute_path(std::string filename)
-{
-#ifdef _WIN32
-	return filename[0] == '/' || filename[0] == '\\' || (filename[0] != 0 && filename[1] == ':');
-#else
-	return filename[0] == '/';
-#endif
-}
-
-void remove_directory(std::string dirname)
-{
-#ifdef _WIN32
-	run_command(stringf("rmdir /s /q \"%s\"", dirname.c_str()));
-#else
-	struct stat stbuf;
-	struct dirent **namelist;
-	int n = scandir(dirname.c_str(), &namelist, nullptr, alphasort);
-	log_assert(n >= 0);
-	for (int i = 0; i < n; i++) {
-		if (strcmp(namelist[i]->d_name, ".") && strcmp(namelist[i]->d_name, "..")) {
-			std::string buffer = stringf("%s/%s", dirname.c_str(), namelist[i]->d_name);
-			if (!stat(buffer.c_str(), &stbuf) && S_ISREG(stbuf.st_mode)) {
-				remove(buffer.c_str());
-			} else
-				remove_directory(buffer);
-		}
-		free(namelist[i]);
-	}
-	free(namelist);
-	rmdir(dirname.c_str());
-#endif
-}
-
-bool create_directory(const std::string& dirname)
-{
-#if defined(_WIN32)
-	int ret = _mkdir(dirname.c_str());
-#else
-	mode_t mode = 0755;
-	int ret = mkdir(dirname.c_str(), mode);
-#endif
-	if (ret == 0)
-		return true;
-
-	switch (errno)
-	{
-	case ENOENT:
-		// parent didn't exist, try to create it
-		{
-			std::string::size_type pos = dirname.find_last_of('/');
-			if (pos == std::string::npos)
-#if defined(_WIN32)
-				pos = dirname.find_last_of('\\');
-			if (pos == std::string::npos)
-#endif
-				return false;
-			if (!create_directory( dirname.substr(0, pos) ))
-				return false;
-		}
-		// now, try to create again
-#if defined(_WIN32)
-		return 0 == _mkdir(dirname.c_str());
-#else
-		return 0 == mkdir(dirname.c_str(), mode);
-#endif
-
-	case EEXIST:
-		// done!
-		return check_directory_exists(dirname);
-
-	default:
-		return false;
-	}
-}
-
-std::string escape_filename_spaces(const std::string& filename)
-{
-	std::string out;
-	out.reserve(filename.size());
-	for (auto c : filename)
-	{
-		if (c == ' ')
-			out += "\\ ";
-		else
-			out.push_back(c);
-	}
-	return out;
-}
-
 bool already_setup = false;
 
 void yosys_setup()

kernel/yosys_common.h

@@ -128,6 +128,7 @@
 #  error "C++17 or later compatible compiler is required"
 #endif
 
+#include "kernel/io.h"
 
 YOSYS_NAMESPACE_BEGIN
 
@@ -245,63 +246,6 @@ inline void memhasher() { if (memhasher_active) memhasher_do(); }
 void yosys_banner();
 int ceil_log2(int x) YS_ATTRIBUTE(const);
 
-inline std::string vstringf(const char *fmt, va_list ap)
-{
-        // For the common case of strings shorter than 128, save a heap
-        // allocation by using a stack allocated buffer.
-        const int kBufSize = 128;
-        char buf[kBufSize];
-        buf[0] = '\0';
-        va_list apc;
-        va_copy(apc, ap);
-        int n = vsnprintf(buf, kBufSize, fmt, apc);
-        va_end(apc);
-        if (n < kBufSize)
-          return std::string(buf);
-
-        std::string string;
-        char *str = NULL;
-#if defined(_WIN32 )|| defined(__CYGWIN__)
-        int sz = 2 * kBufSize, rc;
-        while (1) {
-		va_copy(apc, ap);
-		str = (char*)realloc(str, sz);
-		rc = vsnprintf(str, sz, fmt, apc);
-		va_end(apc);
-		if (rc >= 0 && rc < sz)
-			break;
-		sz *= 2;
-	}
-	if (str != NULL) {
-		string = str;
-		free(str);
-	}
-	return string;
-#else
-        if (vasprintf(&str, fmt, ap) < 0)
-          str = NULL;
-        if (str != NULL) {
-          string = str;
-          free(str);
-        }
-	return string;
-#endif
-}
-
-std::string stringf(const char *fmt, ...) YS_ATTRIBUTE(format(printf, 1, 2));
-
-inline std::string stringf(const char *fmt, ...)
-{
-	std::string string;
-	va_list ap;
-
-	va_start(ap, fmt);
-	string = vstringf(fmt, ap);
-	va_end(ap);
-
-	return string;
-}
-
 int readsome(std::istream &f, char *s, int n);
 std::string next_token(std::string &text, const char *sep = " \t\r\n", bool long_strings = false);
 std::vector<std::string> split_tokens(const std::string &text, const char *sep = " \t\r\n");

passes/cmds/stat.cc

@@ -23,6 +23,7 @@
 #include "kernel/celltypes.h"
 #include "passes/techmap/libparse.h"
 #include "kernel/cost.h"
+#include "kernel/gzip.h"
 #include "libs/json11/json11.hpp"
 
 USING_YOSYS_NAMESPACE
@@ -347,13 +348,12 @@ statdata_t hierarchy_worker(std::map<RTLIL::IdString, statdata_t> &mod_stat, RTL
 
 void read_liberty_cellarea(dict<IdString, cell_area_t> &cell_area, string liberty_file)
 {
-	std::ifstream f;
-	f.open(liberty_file.c_str());
+	std::istream* f = uncompressed(liberty_file.c_str());
 	yosys_input_files.insert(liberty_file);
-	if (f.fail())
+	if (f->fail())
 		log_cmd_error("Can't open liberty file `%s': %s\n", liberty_file.c_str(), strerror(errno));
-	LibertyParser libparser(f);
-	f.close();
+	LibertyParser libparser(*f);
+	delete f;
 
 	for (auto cell : libparser.ast->children)
 	{

passes/fsm/fsm_recode.cc

@@ -22,6 +22,7 @@
 #include "kernel/sigtools.h"
 #include "kernel/consteval.h"
 #include "kernel/celltypes.h"
+#include "kernel/utils.h"
 #include "fsmdata.h"
 #include <math.h>
 #include <string.h>

passes/opt/peepopt.cc

@@ -19,6 +19,7 @@
 
 #include "kernel/yosys.h"
 #include "kernel/sigtools.h"
+#include "kernel/utils.h"
 
 USING_YOSYS_NAMESPACE
 PRIVATE_NAMESPACE_BEGIN

passes/opt/pmux2shiftx.cc

@@ -20,6 +20,7 @@
 #include "kernel/yosys.h"
 #include "kernel/sigtools.h"
 #include "kernel/ffinit.h"
+#include "kernel/utils.h"
 
 USING_YOSYS_NAMESPACE
 PRIVATE_NAMESPACE_BEGIN

passes/opt/wreduce.cc

@@ -21,6 +21,7 @@
 #include "kernel/sigtools.h"
 #include "kernel/modtools.h"
 #include "kernel/ffinit.h"
+#include "kernel/utils.h"
 
 USING_YOSYS_NAMESPACE
 

passes/pmgen/pmgen.py

@@ -354,6 +354,7 @@ with open(outfile, "w") as f:
     if genhdr:
         print("#include \"kernel/yosys.h\"", file=f)
         print("#include \"kernel/sigtools.h\"", file=f)
+        print("#include \"kernel/utils.h\"", file=f)
         print("", file=f)
         print("YOSYS_NAMESPACE_BEGIN", file=f)
         print("", file=f)

passes/sat/assertpmux.cc

@@ -19,6 +19,7 @@
 
 #include "kernel/yosys.h"
 #include "kernel/sigtools.h"
+#include "kernel/utils.h"
 
 USING_YOSYS_NAMESPACE
 PRIVATE_NAMESPACE_BEGIN

passes/techmap/clockgate.cc

@@ -1,5 +1,6 @@
 #include "kernel/yosys.h"
 #include "kernel/ff.h"
+#include "kernel/gzip.h"
 #include "libparse.h"
 #include <optional>
 
@@ -308,13 +309,12 @@ struct ClockgatePass : public Pass {
 		if (!liberty_files.empty()) {
 			LibertyMergedCells merged;
 			for (auto path : liberty_files) {
-				std::ifstream f;
-				f.open(path.c_str());
-				if (f.fail())
+				std::istream* f = uncompressed(path);
+				if (f->fail())
 					log_cmd_error("Can't open liberty file `%s': %s\n", path.c_str(), strerror(errno));
-				LibertyParser p(f);
+				LibertyParser p(*f);
 				merged.merge(p);
-				f.close();
+				delete f;
 			}
 			std::tie(pos_icg_desc, neg_icg_desc) =
 				find_icgs(merged.cells, dont_use_cells);

passes/techmap/dfflibmap.cc

@@ -19,6 +19,7 @@
 
 #include "kernel/yosys.h"
 #include "kernel/sigtools.h"
+#include "kernel/gzip.h"
 #include "libparse.h"
 #include <string.h>
 #include <errno.h>
@@ -630,13 +631,12 @@ struct DfflibmapPass : public Pass {
 
 		LibertyMergedCells merged;
 		for (auto path : liberty_files) {
-			std::ifstream f;
-			f.open(path.c_str());
-			if (f.fail())
+			std::istream* f = uncompressed(path);
+			if (f->fail())
 				log_cmd_error("Can't open liberty file `%s': %s\n", path.c_str(), strerror(errno));
-			LibertyParser p(f);
+			LibertyParser p(*f);
 			merged.merge(p);
-			f.close();
+			delete f;
 		}
 
 		find_cell(merged.cells, ID($_DFF_N_), false, false, false, false, false, false, dont_use_cells);

tests/various/stat.ys

@@ -0,0 +1,14 @@
+read_rtlil << EOF
+module \top
+  wire input 1 \A
+  wire output 2 \Y
+  cell \sg13g2_and2_1 \sub
+    connect \A \A
+    connect \B 1'0
+    connect \Y \Y
+  end
+end
+EOF
+logger -expect log "Chip area for module '\\top': 9.072000" 1
+logger -expect-no-warnings
+stat -liberty ../../tests/liberty/foundry_data/sg13g2_stdcell_typ_1p20V_25C.lib.filtered.gz