Merge pull request #75447 from bruvzg/brotli_packedarray

Expose brotli decompression to the scripting API.

SConstruct

@@ -191,6 +191,7 @@ opts.Add(BoolVariable("production", "Set defaults to build Godot for use in prod
 opts.Add(BoolVariable("deprecated", "Enable compatibility code for deprecated and removed features", True))
 opts.Add(EnumVariable("precision", "Set the floating-point precision level", "single", ("single", "double")))
 opts.Add(BoolVariable("minizip", "Enable ZIP archive support using minizip", True))
+opts.Add(BoolVariable("brotli", "Enable Brotli for decompresson and WOFF2 fonts support", True))
 opts.Add(BoolVariable("xaudio2", "Enable the XAudio2 audio driver", False))
 opts.Add(BoolVariable("vulkan", "Enable the vulkan rendering driver", True))
 opts.Add(BoolVariable("opengl3", "Enable the OpenGL/GLES3 rendering driver", True))
@@ -865,6 +866,8 @@ if selected_platform in platform_list:
             env.Append(CPPDEFINES=["ADVANCED_GUI_DISABLED"])
     if env["minizip"]:
         env.Append(CPPDEFINES=["MINIZIP_ENABLED"])
+    if env["brotli"]:
+        env.Append(CPPDEFINES=["BROTLI_ENABLED"])
 
     if not env["verbose"]:
         methods.no_verbose(sys, env)

core/SCsub

@@ -64,6 +64,31 @@ thirdparty_misc_sources = [
 thirdparty_misc_sources = [thirdparty_misc_dir + file for file in thirdparty_misc_sources]
 env_thirdparty.add_source_files(thirdparty_obj, thirdparty_misc_sources)
 
+# Brotli
+if env["brotli"]:
+    thirdparty_brotli_dir = "#thirdparty/brotli/"
+    thirdparty_brotli_sources = [
+        "common/constants.c",
+        "common/context.c",
+        "common/dictionary.c",
+        "common/platform.c",
+        "common/shared_dictionary.c",
+        "common/transform.c",
+        "dec/bit_reader.c",
+        "dec/decode.c",
+        "dec/huffman.c",
+        "dec/state.c",
+    ]
+    thirdparty_brotli_sources = [thirdparty_brotli_dir + file for file in thirdparty_brotli_sources]
+
+    env_thirdparty.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"])
+    env.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"])
+
+    if env.get("use_ubsan") or env.get("use_asan") or env.get("use_tsan") or env.get("use_lsan") or env.get("use_msan"):
+        env_thirdparty.Append(CPPDEFINES=["BROTLI_BUILD_PORTABLE"])
+
+    env_thirdparty.add_source_files(thirdparty_obj, thirdparty_brotli_sources)
+
 # Zlib library, can be unbundled
 if env["builtin_zlib"]:
     thirdparty_zlib_dir = "#thirdparty/zlib/"

core/io/compression.cpp

@@ -35,11 +35,18 @@
 
 #include "thirdparty/misc/fastlz.h"
 
+#ifdef BROTLI_ENABLED
+#include "thirdparty/brotli/include/brotli/decode.h"
+#endif
+
 #include <zlib.h>
 #include <zstd.h>
 
 int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode) {
 	switch (p_mode) {
+		case MODE_BROTLI: {
+			ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
+		} break;
 		case MODE_FASTLZ: {
 			if (p_src_size < 16) {
 				uint8_t src[16];
@@ -95,6 +102,9 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
 
 int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
 	switch (p_mode) {
+		case MODE_BROTLI: {
+			ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
+		} break;
 		case MODE_FASTLZ: {
 			int ss = p_src_size + p_src_size * 6 / 100;
 			if (ss < 66) {
@@ -129,6 +139,16 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
 
 int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
 	switch (p_mode) {
+		case MODE_BROTLI: {
+#ifdef BROTLI_ENABLED
+			size_t ret_size = p_dst_max_size;
+			BrotliDecoderResult res = BrotliDecoderDecompress(p_src_size, p_src, &ret_size, p_dst);
+			ERR_FAIL_COND_V(res != BROTLI_DECODER_RESULT_SUCCESS, -1);
+			return ret_size;
+#else
+			ERR_FAIL_V_MSG(-1, "Godot was compiled without brotli support.");
+#endif
+		} break;
 		case MODE_FASTLZ: {
 			int ret_size = 0;
 
@@ -186,87 +206,147 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
 	This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
 */
 int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
-	int ret;
 	uint8_t *dst = nullptr;
 	int out_mark = 0;
-	z_stream strm;
 
 	ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
 
-	// This function only supports GZip and Deflate
-	int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
-	ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
+	if (p_mode == MODE_BROTLI) {
+#ifdef BROTLI_ENABLED
+		BrotliDecoderResult ret;
+		BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+		ERR_FAIL_COND_V(state == nullptr, Z_DATA_ERROR);
 
-	// Initialize the stream
-	strm.zalloc = Z_NULL;
-	strm.zfree = Z_NULL;
-	strm.opaque = Z_NULL;
-	strm.avail_in = 0;
-	strm.next_in = Z_NULL;
+		// Setup the stream inputs.
+		const uint8_t *next_in = p_src;
+		size_t avail_in = p_src_size;
+		uint8_t *next_out = nullptr;
+		size_t avail_out = 0;
+		size_t total_out = 0;
 
-	int err = inflateInit2(&strm, window_bits);
-	ERR_FAIL_COND_V(err != Z_OK, -1);
+		// Ensure the destination buffer is empty.
+		p_dst_vect->clear();
 
-	// Setup the stream inputs
-	strm.next_in = (Bytef *)p_src;
-	strm.avail_in = p_src_size;
-
-	// Ensure the destination buffer is empty
-	p_dst_vect->clear();
-
-	// decompress until deflate stream ends or end of file
-	do {
-		// Add another chunk size to the output buffer
-		// This forces a copy of the whole buffer
-		p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
-		// Get pointer to the actual output buffer
-		dst = p_dst_vect->ptrw();
-
-		// Set the stream to the new output stream
-		// Since it was copied, we need to reset the stream to the new buffer
-		strm.next_out = &(dst[out_mark]);
-		strm.avail_out = gzip_chunk;
-
-		// run inflate() on input until output buffer is full and needs to be resized
-		// or input runs out
+		// Decompress until stream ends or end of file.
 		do {
-			ret = inflate(&strm, Z_SYNC_FLUSH);
+			// Add another chunk size to the output buffer.
+			// This forces a copy of the whole buffer.
+			p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
+			// Get pointer to the actual output buffer.
+			dst = p_dst_vect->ptrw();
 
-			switch (ret) {
-				case Z_NEED_DICT:
-					ret = Z_DATA_ERROR;
-					[[fallthrough]];
-				case Z_DATA_ERROR:
-				case Z_MEM_ERROR:
-				case Z_STREAM_ERROR:
-				case Z_BUF_ERROR:
-					if (strm.msg) {
-						WARN_PRINT(strm.msg);
-					}
-					(void)inflateEnd(&strm);
-					p_dst_vect->clear();
-					return ret;
+			// Set the stream to the new output stream.
+			// Since it was copied, we need to reset the stream to the new buffer.
+			next_out = &(dst[out_mark]);
+			avail_out += gzip_chunk;
+
+			ret = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, &total_out);
+			if (ret == BROTLI_DECODER_RESULT_ERROR) {
+				WARN_PRINT(BrotliDecoderErrorString(BrotliDecoderGetErrorCode(state)));
+				BrotliDecoderDestroyInstance(state);
+				p_dst_vect->clear();
+				return Z_DATA_ERROR;
 			}
-		} while (strm.avail_out > 0 && strm.avail_in > 0);
 
-		out_mark += gzip_chunk;
+			out_mark += gzip_chunk - avail_out;
 
-		// Enforce max output size
-		if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
-			(void)inflateEnd(&strm);
-			p_dst_vect->clear();
-			return Z_BUF_ERROR;
+			// Enforce max output size.
+			if (p_max_dst_size > -1 && total_out > (uint64_t)p_max_dst_size) {
+				BrotliDecoderDestroyInstance(state);
+				p_dst_vect->clear();
+				return Z_BUF_ERROR;
+			}
+		} while (ret != BROTLI_DECODER_RESULT_SUCCESS);
+
+		// If all done successfully, resize the output if it's larger than the actual output.
+		if ((unsigned long)p_dst_vect->size() > total_out) {
+			p_dst_vect->resize(total_out);
 		}
-	} while (ret != Z_STREAM_END);
 
-	// If all done successfully, resize the output if it's larger than the actual output
-	if ((unsigned long)p_dst_vect->size() > strm.total_out) {
-		p_dst_vect->resize(strm.total_out);
+		// Clean up and return.
+		BrotliDecoderDestroyInstance(state);
+		return Z_OK;
+#else
+		ERR_FAIL_V_MSG(Z_ERRNO, "Godot was compiled without brotli support.");
+#endif
+	} else {
+		// This function only supports GZip and Deflate.
+		ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
+
+		int ret;
+		z_stream strm;
+		int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+
+		// Initialize the stream.
+		strm.zalloc = Z_NULL;
+		strm.zfree = Z_NULL;
+		strm.opaque = Z_NULL;
+		strm.avail_in = 0;
+		strm.next_in = Z_NULL;
+
+		int err = inflateInit2(&strm, window_bits);
+		ERR_FAIL_COND_V(err != Z_OK, -1);
+
+		// Setup the stream inputs.
+		strm.next_in = (Bytef *)p_src;
+		strm.avail_in = p_src_size;
+
+		// Ensure the destination buffer is empty.
+		p_dst_vect->clear();
+
+		// Decompress until deflate stream ends or end of file.
+		do {
+			// Add another chunk size to the output buffer.
+			// This forces a copy of the whole buffer.
+			p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
+			// Get pointer to the actual output buffer.
+			dst = p_dst_vect->ptrw();
+
+			// Set the stream to the new output stream.
+			// Since it was copied, we need to reset the stream to the new buffer.
+			strm.next_out = &(dst[out_mark]);
+			strm.avail_out = gzip_chunk;
+
+			// Run inflate() on input until output buffer is full and needs to be resized or input runs out.
+			do {
+				ret = inflate(&strm, Z_SYNC_FLUSH);
+
+				switch (ret) {
+					case Z_NEED_DICT:
+						ret = Z_DATA_ERROR;
+						[[fallthrough]];
+					case Z_DATA_ERROR:
+					case Z_MEM_ERROR:
+					case Z_STREAM_ERROR:
+					case Z_BUF_ERROR:
+						if (strm.msg) {
+							WARN_PRINT(strm.msg);
+						}
+						(void)inflateEnd(&strm);
+						p_dst_vect->clear();
+						return ret;
+				}
+			} while (strm.avail_out > 0 && strm.avail_in > 0);
+
+			out_mark += gzip_chunk;
+
+			// Enforce max output size.
+			if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
+				(void)inflateEnd(&strm);
+				p_dst_vect->clear();
+				return Z_BUF_ERROR;
+			}
+		} while (ret != Z_STREAM_END);
+
+		// If all done successfully, resize the output if it's larger than the actual output.
+		if ((unsigned long)p_dst_vect->size() > strm.total_out) {
+			p_dst_vect->resize(strm.total_out);
+		}
+
+		// Clean up and return.
+		(void)inflateEnd(&strm);
+		return Z_OK;
 	}
-
-	// clean up and return
-	(void)inflateEnd(&strm);
-	return Z_OK;
 }
 
 int Compression::zlib_level = Z_DEFAULT_COMPRESSION;

core/io/compression.h

@@ -47,7 +47,8 @@ public:
 		MODE_FASTLZ,
 		MODE_DEFLATE,
 		MODE_ZSTD,
-		MODE_GZIP
+		MODE_GZIP,
+		MODE_BROTLI
 	};
 
 	static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);

core/io/file_access.cpp

@@ -871,4 +871,5 @@ void FileAccess::_bind_methods() {
 	BIND_ENUM_CONSTANT(COMPRESSION_DEFLATE);
 	BIND_ENUM_CONSTANT(COMPRESSION_ZSTD);
 	BIND_ENUM_CONSTANT(COMPRESSION_GZIP);
+	BIND_ENUM_CONSTANT(COMPRESSION_BROTLI);
 }

core/io/file_access.h

@@ -64,7 +64,8 @@ public:
 		COMPRESSION_FASTLZ = Compression::MODE_FASTLZ,
 		COMPRESSION_DEFLATE = Compression::MODE_DEFLATE,
 		COMPRESSION_ZSTD = Compression::MODE_ZSTD,
-		COMPRESSION_GZIP = Compression::MODE_GZIP
+		COMPRESSION_GZIP = Compression::MODE_GZIP,
+		COMPRESSION_BROTLI = Compression::MODE_BROTLI,
 	};
 
 	typedef void (*FileCloseFailNotify)(const String &);

doc/classes/FileAccess.xml

@@ -492,5 +492,8 @@
 		<constant name="COMPRESSION_GZIP" value="3" enum="CompressionMode">
 			Uses the [url=https://www.gzip.org/]gzip[/url] compression method.
 		</constant>
+		<constant name="COMPRESSION_BROTLI" value="4" enum="CompressionMode">
+			Uses the [url=https://github.com/google/brotli]brotli[/url] compression method (only decompression is supported).
+		</constant>
 	</constants>
 </class>

doc/classes/PackedByteArray.xml

@@ -181,7 +181,7 @@
 			<param index="0" name="max_output_size" type="int" />
 			<param index="1" name="compression_mode" type="int" default="0" />
 			<description>
-				Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum FileAccess.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b]
+				Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum FileAccess.CompressionMode]'s constants. [b]This method only accepts brotli, gzip, and deflate compression modes.[/b]
 				This method is potentially slower than [code]decompress[/code], as it may have to re-allocate its output buffer multiple times while decompressing, whereas [code]decompress[/code] knows it's output buffer size from the beginning.
 				GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [param max_output_size]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that amount in bytes, then an error will be returned.
 			</description>

modules/freetype/SCsub

@@ -59,25 +59,7 @@ if env["builtin_freetype"]:
     thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources]
 
     if env["brotli"]:
-        thirdparty_brotli_dir = "#thirdparty/brotli/"
-        thirdparty_brotli_sources = [
-            "common/constants.c",
-            "common/context.c",
-            "common/dictionary.c",
-            "common/platform.c",
-            "common/shared_dictionary.c",
-            "common/transform.c",
-            "dec/bit_reader.c",
-            "dec/decode.c",
-            "dec/huffman.c",
-            "dec/state.c",
-        ]
-        thirdparty_sources += [thirdparty_brotli_dir + file for file in thirdparty_brotli_sources]
         env_freetype.Append(CPPDEFINES=["FT_CONFIG_OPTION_USE_BROTLI"])
-        env_freetype.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"])
-
-    if env.get("use_ubsan") or env.get("use_asan") or env.get("use_tsan") or env.get("use_lsan") or env.get("use_msan"):
-        env_freetype.Append(CPPDEFINES=["BROTLI_BUILD_PORTABLE"])
 
     if env["platform"] == "uwp":
         # Include header for UWP to fix build issues

modules/freetype/config.py

@@ -2,13 +2,5 @@ def can_build(env, platform):
     return True
 
 
-def get_opts(platform):
-    from SCons.Variables import BoolVariable
-
-    return [
-        BoolVariable("brotli", "Enable Brotli decompressor for WOFF2 fonts support", True),
-    ]
-
-
 def configure(env):
     pass