Merge pull request #14467 from unknownbrackets/texreplace-zim

Add support for ZIMs in texture replacements

Common/Data/Format/ZIMSave.cpp

@@ -24,7 +24,7 @@ static unsigned int log2i(unsigned int val) {
 }
 
 
-int ezcompress(unsigned char* pDest, long* pnDestLen, const unsigned char* pSrc, long nSrcLen) {
+int ezcompress(unsigned char* pDest, long* pnDestLen, const unsigned char* pSrc, long nSrcLen, int compressLevel) {
 	z_stream stream;
 	int err;
 
@@ -43,7 +43,7 @@ int ezcompress(unsigned char* pDest, long* pnDestLen, const unsigned char* pSrc,
 	stream.zfree = (free_func)0;
 	stream.opaque = (voidpf)0;
 
-	err = deflateInit(&stream, Z_DEFAULT_COMPRESSION);
+	err = deflateInit(&stream, compressLevel);
 	if (err != Z_OK) return err;
 	nExtraChunks = 0;
 	do {
@@ -179,7 +179,7 @@ uint8_t *DownsampleBy2(const uint8_t *image, int width, int height, int pitch) {
 	return out;
 }
 
-void SaveZIM(FILE *f, int width, int height, int pitch, int flags, const uint8_t *image_data) {
+void SaveZIM(FILE *f, int width, int height, int pitch, int flags, const uint8_t *image_data, int compressLevel) {
 	fwrite(magic, 1, 4, f);
 	fwrite(&width, 1, 4, f);
 	fwrite(&height, 1, 4, f);
@@ -196,7 +196,7 @@ void SaveZIM(FILE *f, int width, int height, int pitch, int flags, const uint8_t
 		if (flags & ZIM_ZLIB_COMPRESSED) {
 			long dest_len = data_size * 2;
 			uint8_t *dest = new uint8_t[dest_len];
-			if (Z_OK == ezcompress(dest, &dest_len, data, data_size)) {
+			if (Z_OK == ezcompress(dest, &dest_len, data, data_size, compressLevel == 0 ? Z_DEFAULT_COMPRESSION : compressLevel)) {
 				fwrite(dest, 1, dest_len, f);
 			} else {
 				ERROR_LOG(IO, "Zlib compression failed.\n");
@@ -205,7 +205,7 @@ void SaveZIM(FILE *f, int width, int height, int pitch, int flags, const uint8_t
 		} else if (flags & ZIM_ZSTD_COMPRESSED) {
 			size_t dest_len = ZSTD_compressBound(data_size);
 			uint8_t *dest = new uint8_t[dest_len];
-			dest_len = ZSTD_compress(dest, dest_len, data, data_size, 22);
+			dest_len = ZSTD_compress(dest, dest_len, data, data_size, compressLevel == 0 ? 22 : compressLevel);
 			if (!ZSTD_isError(dest_len)) {
 				fwrite(dest, 1, dest_len, f);
 			} else {

Common/Data/Format/ZIMSave.h

@@ -11,4 +11,4 @@
 // * Generate mipmaps if requested
 // * Convert images to the requested format
 // Input image is always 8888 RGBA. SaveZIM takes care of downsampling and mipmap generation.
-void SaveZIM(FILE *f, int width, int height, int pitch, int format, const uint8_t *image);
+void SaveZIM(FILE *f, int width, int height, int pitch, int format, const uint8_t *image, int compressLevel = 0);

Core/TextureReplacer.cpp

@@ -17,12 +17,15 @@
 
 #include "ppsspp_config.h"
 #include <algorithm>
+#include <cstring>
+#include <memory>
 #include <png.h>
 
 #include "ext/xxhash.h"
 
 #include "Common/Data/Convert/ColorConv.h"
 #include "Common/Data/Format/IniFile.h"
+#include "Common/Data/Format/ZIMLoad.h"
 #include "Common/Data/Text/I18n.h"
 #include "Common/Data/Text/Parsers.h"
 #include "Common/File/FileUtil.h"
@@ -31,6 +34,7 @@
 #include "Core/Host.h"
 #include "Core/System.h"
 #include "Core/TextureReplacer.h"
+#include "Core/ThreadPools.h"
 #include "Core/ELF/ParamSFO.h"
 #include "GPU/Common/TextureDecoder.h"
 
@@ -397,25 +401,14 @@ void TextureReplacer::PopulateReplacement(ReplacedTexture *result, u64 cachekey,
 			break;
 		}
 
-		bool good = false;
 		ReplacedTextureLevel level;
 		level.fmt = ReplacedTextureFormat::F_8888;
 		level.file = filename;
+		bool good = PopulateLevel(level);
 
-		png_image png = {};
-		png.version = PNG_IMAGE_VERSION;
-		FILE *fp = File::OpenCFile(filename, "rb");
-		if (png_image_begin_read_from_stdio(&png, fp)) {
-			// We pad files that have been hashrange'd so they are the same texture size.
-			level.w = (png.width * w) / newW;
-			level.h = (png.height * h) / newH;
-			good = true;
-		} else {
-			ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s", filename.c_str(), png.message);
-		}
-		fclose(fp);
-
-		png_image_free(&png);
+		// We pad files that have been hashrange'd so they are the same texture size.
+		level.w = (level.w * w) / newW;
+		level.h = (level.h * h) / newH;
 
 		if (good && i != 0) {
 			// Check that the mipmap size is correct.  Can't load mips of the wrong size.
@@ -435,6 +428,58 @@ void TextureReplacer::PopulateReplacement(ReplacedTexture *result, u64 cachekey,
 	result->alphaStatus_ = ReplacedTextureAlpha::UNKNOWN;
 }
 
+enum class ReplacedImageType {
+	PNG,
+	ZIM,
+	INVALID,
+};
+
+static ReplacedImageType Identify(FILE *fp) {
+	uint8_t magic[4];
+	if (fread(magic, 1, 4, fp) != 4)
+		return ReplacedImageType::INVALID;
+	rewind(fp);
+
+	if (strncmp((const char *)magic, "ZIMG", 4) == 0)
+		return ReplacedImageType::ZIM;
+	if (magic[0] == 0x89 && strncmp((const char *)&magic[1], "PNG", 3) == 0)
+		return ReplacedImageType::PNG;
+	return ReplacedImageType::INVALID;
+}
+
+bool TextureReplacer::PopulateLevel(ReplacedTextureLevel &level) {
+	bool good = false;
+
+	FILE *fp = File::OpenCFile(level.file, "rb");
+	auto imageType = Identify(fp);
+	if (imageType == ReplacedImageType::ZIM) {
+		fseek(fp, 4, SEEK_SET);
+		fread(&level.w, 1, 4, fp);
+		fread(&level.h, 1, 4, fp);
+		int flags;
+		if (fread(&flags, 1, 4, fp) == 4) {
+			good = (flags & ZIM_FORMAT_MASK) == ZIM_RGBA8888;
+		}
+	} else if (imageType == ReplacedImageType::PNG) {
+		png_image png = {};
+		png.version = PNG_IMAGE_VERSION;
+		if (png_image_begin_read_from_stdio(&png, fp)) {
+			// We pad files that have been hashrange'd so they are the same texture size.
+			level.w = png.width;
+			level.h = png.height;
+			good = true;
+		} else {
+			ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s", level.file.ToVisualString().c_str(), png.message);
+		}
+		png_image_free(&png);
+	} else {
+		ERROR_LOG(G3D, "Could not load texture replacement info: %s - unsupported format", level.file.ToVisualString().c_str());
+	}
+	fclose(fp);
+
+	return good;
+}
+
 static bool WriteTextureToPNG(png_imagep image, const Path &filename, int convert_to_8bit, const void *buffer, png_int_32 row_stride, const void *colormap) {
 	FILE *fp = File::OpenCFile(filename, "wb");
 	if (!fp) {
@@ -693,40 +738,64 @@ void ReplacedTexture::Load(int level, void *out, int rowPitch) {
 
 	const ReplacedTextureLevel &info = levels_[level];
 
-	png_image png = {};
-	png.version = PNG_IMAGE_VERSION;
-
 	FILE *fp = File::OpenCFile(info.file, "rb");
-	if (!png_image_begin_read_from_stdio(&png, fp)) {
-		ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s", info.file.c_str(), png.message);
-		return;
-	}
-
-	bool checkedAlpha = false;
-	if ((png.format & PNG_FORMAT_FLAG_ALPHA) == 0) {
-		// Well, we know for sure it doesn't have alpha.
-		if (level == 0) {
-			alphaStatus_ = ReplacedTextureAlpha::FULL;
+	auto imageType = Identify(fp);
+	if (imageType == ReplacedImageType::ZIM) {
+		size_t zimSize = File::GetFileSize(fp);
+		std::unique_ptr<uint8_t[]> zim(new uint8_t[zimSize]);
+		fread(&zim[0], 1, zimSize, fp);
+
+		int w, h, f;
+		uint8_t *image;
+		if (LoadZIMPtr(&zim[0], zimSize, &w, &h, &f, &image)) {
+			GlobalThreadPool::Loop([&](int low, int high) {
+				for (int y = low; y < high; ++y) {
+					memcpy((uint8_t *)out + rowPitch * y, image + w * 4 * y, w * 4);
+				}
+			}, 0, h);
+			free(image);
 		}
-		checkedAlpha = true;
-	}
-	png.format = PNG_FORMAT_RGBA;
-
-	if (!png_image_finish_read(&png, nullptr, out, rowPitch, nullptr)) {
-		ERROR_LOG(G3D, "Could not load texture replacement: %s - %s", info.file.c_str(), png.message);
-		return;
-	}
 
-	if (!checkedAlpha) {
 		// This will only check the hashed bits.
-		CheckAlphaResult res = CheckAlphaRGBA8888Basic((u32 *)out, rowPitch / sizeof(u32), png.width, png.height);
+		CheckAlphaResult res = CheckAlphaRGBA8888Basic((u32 *)out, rowPitch / sizeof(u32), w, h);
 		if (res == CHECKALPHA_ANY || level == 0) {
 			alphaStatus_ = ReplacedTextureAlpha(res);
 		}
+	} else if (imageType == ReplacedImageType::PNG) {
+		png_image png = {};
+		png.version = PNG_IMAGE_VERSION;
+
+		if (!png_image_begin_read_from_stdio(&png, fp)) {
+			ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s", info.file.c_str(), png.message);
+			return;
+		}
+
+		bool checkedAlpha = false;
+		if ((png.format & PNG_FORMAT_FLAG_ALPHA) == 0) {
+			// Well, we know for sure it doesn't have alpha.
+			if (level == 0) {
+				alphaStatus_ = ReplacedTextureAlpha::FULL;
+			}
+			checkedAlpha = true;
+		}
+		png.format = PNG_FORMAT_RGBA;
+
+		if (!png_image_finish_read(&png, nullptr, out, rowPitch, nullptr)) {
+			ERROR_LOG(G3D, "Could not load texture replacement: %s - %s", info.file.c_str(), png.message);
+			return;
+		}
+		png_image_free(&png);
+
+		if (!checkedAlpha) {
+			// This will only check the hashed bits.
+			CheckAlphaResult res = CheckAlphaRGBA8888Basic((u32 *)out, rowPitch / sizeof(u32), png.width, png.height);
+			if (res == CHECKALPHA_ANY || level == 0) {
+				alphaStatus_ = ReplacedTextureAlpha(res);
+			}
+		}
 	}
 
 	fclose(fp);
-	png_image_free(&png);
 }
 
 bool TextureReplacer::GenerateIni(const std::string &gameID, Path &generatedFilename) {

Core/TextureReplacer.h

@@ -204,6 +204,7 @@ class TextureReplacer {
 	std::string LookupHashFile(u64 cachekey, u32 hash, int level);
 	std::string HashName(u64 cachekey, u32 hash, int level);
 	void PopulateReplacement(ReplacedTexture *result, u64 cachekey, u32 hash, int w, int h);
+	bool PopulateLevel(ReplacedTextureLevel &level);
 
 	SimpleBuf<u32> saveBuf;
 	bool enabled_ = false;

ext/native/tools/zimtool.cpp

@@ -14,7 +14,7 @@ const char *format_strings[4] = { "8888", "4444", "565", "ETC1" };
 int formats[3] = { ZIM_RGBA8888, ZIM_RGBA4444, ZIM_RGB565 };
 
 void printusage() {
-	fprintf(stderr, "Usage: zimtool infile.png outfile.zim [-f=FORMAT] [-m] [-g]\n");
+	fprintf(stderr, "Usage: zimtool infile.png outfile.zim [-f=FORMAT] [-m] [-g] [-z[LEVEL]]\n");
 	fprintf(stderr, "Formats: 8888 4444 565 ETC1\n");
 }
 
@@ -40,6 +40,7 @@ int main(int argc, char **argv) {
 	}
 
 	int flags = 0;
+	int level = 0;
 	bool format_set = false;
 	for (int i = 3; i < argc; i++) {
 		if (argv[i][0] != '-') {
@@ -68,6 +69,16 @@ int main(int argc, char **argv) {
 			}
 		}
 		break;
+		case 'z':
+			flags |= ZIM_ZSTD_COMPRESSED;
+			if (argv[i][2] != '\0') {
+				int pos = 2;
+				while (argv[i][pos] >= '0' && argv[i][pos] <= '9') {
+					level = level * 10 + argv[i][pos] - '0';
+					pos++;
+				}
+			}
+			break;
 		}
 	}
 	// TODO: make setting?
@@ -87,7 +98,7 @@ int main(int argc, char **argv) {
 	}
 
 	FILE *f = fopen(FLAGS_outfile, "wb");
-	SaveZIM(f, width, height, width * 4, flags, image_data);
+	SaveZIM(f, width, height, width * 4, flags, image_data, level);
 	fclose(f);
 	int in_file_size = filesize(FLAGS_infile);
 	int out_file_size = filesize(FLAGS_outfile);