Memory-efficient zlib usage across Liberty file consumers

Makefile

@@ -578,7 +578,9 @@ $(eval $(call add_include_file,kernel/fmt.h))
 ifeq ($(ENABLE_ZLIB),1)
 $(eval $(call add_include_file,kernel/fstdata.h))
 endif
+$(eval $(call add_include_file,kernel/gzip.h))
 $(eval $(call add_include_file,kernel/hashlib.h))
+$(eval $(call add_include_file,kernel/io.h))
 $(eval $(call add_include_file,kernel/json.h))
 $(eval $(call add_include_file,kernel/log.h))
 $(eval $(call add_include_file,kernel/macc.h))
@@ -609,7 +611,7 @@ $(eval $(call add_include_file,frontends/ast/ast_binding.h))
 $(eval $(call add_include_file,frontends/blif/blifparse.h))
 $(eval $(call add_include_file,backends/rtlil/rtlil_backend.h))
 
-OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o
+OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o kernel/io.o kernel/gzip.o
 OBJS += kernel/binding.o kernel/tclapi.o
 OBJS += kernel/cellaigs.o kernel/celledges.o kernel/cost.o kernel/satgen.o kernel/scopeinfo.o kernel/qcsat.o kernel/mem.o kernel/ffmerge.o kernel/ff.o kernel/yw.o kernel/json.o kernel/fmt.o kernel/sexpr.o
 OBJS += kernel/drivertools.o kernel/functional.o

backends/btor/btor.cc

@@ -30,6 +30,7 @@
 #include "kernel/mem.h"
 #include "kernel/json.h"
 #include "kernel/yw.h"
+#include "kernel/utils.h"
 #include <string>
 
 USING_YOSYS_NAMESPACE

backends/smt2/smt2.cc

@@ -24,6 +24,7 @@
 #include "kernel/log.h"
 #include "kernel/mem.h"
 #include "libs/json11/json11.hpp"
+#include "kernel/utils.h"
 #include <string>
 
 USING_YOSYS_NAMESPACE

frontends/verific/verific.cc

@@ -21,6 +21,7 @@
 #include "kernel/sigtools.h"
 #include "kernel/celltypes.h"
 #include "kernel/log.h"
+#include "kernel/utils.h"
 #include "libs/sha1/sha1.h"
 #include <stdlib.h>
 #include <stdio.h>

kernel/gzip.cc

@@ -0,0 +1,139 @@
+#include "kernel/yosys_common.h"
+#include "kernel/log.h"
+#include "kernel/gzip.h"
+#include <iostream>
+#include <string>
+#include <cstdarg>
+#include <cstdio>
+
+#if !defined(WIN32)
+#include <dirent.h>
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+
+YOSYS_NAMESPACE_BEGIN
+
+#ifdef YOSYS_ENABLE_ZLIB
+
+gzip_ostream::obuf::obuf() {
+    setp(buffer, buffer + buffer_size - 1);
+}
+
+bool gzip_ostream::obuf::open(const std::string &filename) {
+    gzf = Zlib::gzopen(filename.c_str(), "wb");
+    return gzf != nullptr;
+}
+
+int gzip_ostream::obuf::sync() {
+    int num = pptr() - pbase();
+    if (num > 0) {
+        if (Zlib::gzwrite(gzf, reinterpret_cast<const void*>(pbase()), num) != num) {
+            return -1;
+        }
+        pbump(-num);
+    }
+    return 0;
+}
+
+gzip_ostream::obuf::~obuf() {
+    if (gzf) {
+        sync();
+        Zlib::gzclose(gzf);
+    }
+}
+
+bool gzip_istream::ibuf::open(const std::string& filename) {
+	if (gzf) {
+		Zlib::gzclose(gzf);
+	}
+	gzf = Zlib::gzopen(filename.c_str(), "rb");
+	if (!gzf) {
+		return false;
+	}
+	// Empty and point to start
+	setg(buffer, buffer, buffer);
+	return true;
+}
+
+// Called when the buffer is empty and more input is needed
+std::istream::int_type gzip_istream::ibuf::underflow() {
+	log_assert(gzf && "No gzfile opened\n");
+	int bytes_read = Zlib::gzread(gzf, buffer, buffer_size);
+	if (bytes_read <= 0) {
+		if (Zlib::gzeof(gzf)) {
+			// "On failure, the function ensures that either
+			// gptr() == nullptr or gptr() == egptr."
+			// Let's set gptr to egptr
+			setg(eback(), egptr(), egptr());
+			return traits_type::eof();
+		}
+
+		int err;
+		const char* error_msg = Zlib::gzerror(gzf, &err);
+		if (err != Z_OK)
+			log_error("%s", error_msg);
+		else
+			log_error("Decompression logic failure: "\
+					"read <=0 bytes but neither EOF nor error\n");
+	}
+
+	// Keep size and point to start
+	setg(buffer, buffer, buffer + bytes_read);
+	return traits_type::to_int_type(buffer[0]);
+}
+
+gzip_istream::ibuf::~ibuf() {
+	if (gzf) {
+		int err = Zlib::gzclose(gzf);
+		if (err != Z_OK) {
+			// OK to overwrite rr it, it doesn't change
+			const char* error_msg = Zlib::gzerror(gzf, &err);
+			log_error("%s", error_msg);
+		}
+	}
+}
+
+#endif // YOSYS_ENABLE_ZLIB
+
+
+// Takes a successfully opened ifstream. If it's gzipped, returns an istream. Otherwise,
+// returns the original ifstream, rewound to the start.
+std::istream* uncompressed(const std::string filename, std::ios_base::openmode mode) {
+	std::ifstream* f = new std::ifstream();
+	f->open(filename, mode);
+	if (f->fail())
+		return f;
+	// Check for gzip magic
+	unsigned char magic[3];
+	int n = 0;
+	while (n < 3)
+	{
+		int c = f->get();
+		if (c != EOF) {
+			magic[n] = (unsigned char) c;
+		}
+		n++;
+	}
+	if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) {
+#ifdef YOSYS_ENABLE_ZLIB
+		log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str());
+		if (magic[2] != 8)
+			log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n",
+				filename.c_str(), unsigned(magic[2]));
+		gzip_istream* s = new gzip_istream();
+		delete f;
+		s->open(filename.c_str());
+		return s;
+#else
+		log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str());
+#endif // YOSYS_ENABLE_ZLIB
+	} else {
+		f->clear();
+		f->seekg(0, std::ios::beg);
+		return f;
+	}
+}
+
+YOSYS_NAMESPACE_END

kernel/gzip.h

@@ -0,0 +1,78 @@
+#include <string>
+#include "kernel/yosys_common.h"
+
+#ifndef YOSYS_GZIP_H
+#define YOSYS_GZIP_H
+
+YOSYS_NAMESPACE_BEGIN
+
+#ifdef YOSYS_ENABLE_ZLIB
+
+namespace Zlib {
+#include <zlib.h>
+}
+
+/*
+An output stream that uses a stringbuf to buffer data internally,
+using zlib to write gzip-compressed data every time the stream is flushed.
+*/
+class gzip_ostream : public std::ostream {
+public:
+	gzip_ostream(): std::ostream(nullptr) {
+		rdbuf(&outbuf);
+	}
+	bool open(const std::string &filename) {
+		return outbuf.open(filename);
+	}
+private:
+	class obuf : public std::stringbuf {
+	public:
+		obuf();
+		bool open(const std::string &filename);
+		virtual int sync() override;
+		virtual ~obuf();
+	private:
+		static const int buffer_size = 4096;
+		char buffer[buffer_size];             // Internal buffer for compressed data
+		Zlib::gzFile gzf = nullptr;                 // Handle to the gzip file
+	};
+
+	obuf outbuf;  // The stream buffer instance
+};
+
+/*
+An input stream that uses zlib to read gzip-compressed data from a file,
+buffering the decompressed data internally using its own buffer.
+*/
+class gzip_istream final : public std::istream {
+public:
+	gzip_istream() : std::istream(&inbuf) {}
+	bool open(const std::string& filename) {
+		return inbuf.open(filename);
+	}
+private:
+	class ibuf final : public std::streambuf {
+	public:
+		ibuf() : gzf(nullptr) {}
+		bool open(const std::string& filename);
+		virtual ~ibuf();
+
+	protected:
+		// Called when the buffer is empty and more input is needed
+		virtual int_type underflow() override;
+	private:
+		static const int buffer_size = 8192;
+		char buffer[buffer_size];
+		Zlib::gzFile gzf;
+	};
+
+	ibuf inbuf;  // The stream buffer instance
+};
+
+#endif // YOSYS_ENABLE_ZLIB
+
+std::istream* uncompressed(const std::string filename, std::ios_base::openmode mode = std::ios_base::in);
+
+YOSYS_NAMESPACE_END
+
+#endif // YOSYS_GZIP_H