mirror of
https://github.com/lighttransport/tinyusdz.git
synced 2026-01-18 01:11:17 +01:00
Add zstd compression support for USD files
Add file-level zstd compression/decompression for USD files (USDA, USDC, USDZ). This allows transparent loading of zstd-compressed USD files and optional compression when writing. Features: - Automatic detection by zstd magic number (0x28 0xB5 0x2F 0xFD) - Transparent decompression in LoadUSDFromMemory() - Optional compression in SaveAsUSDA() and SaveAsUSDCToFile() - Auto-detection of .zst file extension for compression - Configurable compression level (1-22, default 5) - Memory budget enforcement during decompression - Uses amalgamated single-file zstd library (v1.5.6) Build options: - TINYUSDZ_WITH_ZSTD_COMPRESSION=ON (default) - TINYUSDZ_USE_SYSTEM_ZSTD=ON (use system library) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -186,6 +186,11 @@ endif ()
|
|||||||
option(TINYUSDZ_USE_SYSTEM_ZLIB
|
option(TINYUSDZ_USE_SYSTEM_ZLIB
|
||||||
"Use system's zlib instead of miniz for TinyEXR/TIFF" OFF)
|
"Use system's zlib instead of miniz for TinyEXR/TIFF" OFF)
|
||||||
|
|
||||||
|
# -- Zstd compression --
|
||||||
|
option(TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
"Enable zstd compression for USD files" ON)
|
||||||
|
option(TINYUSDZ_USE_SYSTEM_ZSTD
|
||||||
|
"Use system's zstd instead of bundled version" OFF)
|
||||||
|
|
||||||
option(
|
option(
|
||||||
TINYUSDZ_PRODUCTION_BUILD
|
TINYUSDZ_PRODUCTION_BUILD
|
||||||
@@ -667,6 +672,21 @@ if(TINYUSDZ_WITH_TIFF OR TINYUSDZ_WITH_EXR)
|
|||||||
|
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
# -- Zstd compression --
|
||||||
|
if(TINYUSDZ_WITH_ZSTD_COMPRESSION)
|
||||||
|
if(NOT TINYUSDZ_USE_SYSTEM_ZSTD)
|
||||||
|
list(APPEND TINYUSDZ_DEP_SOURCES ${PROJECT_SOURCE_DIR}/src/external/zstd.c)
|
||||||
|
# Disable ASM to avoid linker issues, suppress warnings for third-party code
|
||||||
|
set_source_files_properties(
|
||||||
|
${PROJECT_SOURCE_DIR}/src/external/zstd.c
|
||||||
|
PROPERTIES COMPILE_DEFINITIONS "ZSTD_DISABLE_ASM=1"
|
||||||
|
COMPILE_FLAGS "-w")
|
||||||
|
else()
|
||||||
|
find_package(zstd REQUIRED)
|
||||||
|
endif()
|
||||||
|
list(APPEND TINYUSDZ_DEP_SOURCES ${PROJECT_SOURCE_DIR}/src/zstd-compression.cc)
|
||||||
|
endif()
|
||||||
|
|
||||||
if(TINYUSDZ_WITH_ALAC_AUDIO)
|
if(TINYUSDZ_WITH_ALAC_AUDIO)
|
||||||
list(APPEND TINYUSDZ_DEP_SOURCES
|
list(APPEND TINYUSDZ_DEP_SOURCES
|
||||||
${PROJECT_SOURCE_DIR}/src/external/alac/codec/EndianPortable.c
|
${PROJECT_SOURCE_DIR}/src/external/alac/codec/EndianPortable.c
|
||||||
@@ -1341,6 +1361,13 @@ foreach(TINYUSDZ_LIB_TARGET ${TINYUSDZ_LIBS})
|
|||||||
target_compile_definitions(${TINYUSDZ_LIB_TARGET} PRIVATE "TINYUSDZ_NO_STB_IMAGE_WRITE_IMPLEMENTATION")
|
target_compile_definitions(${TINYUSDZ_LIB_TARGET} PRIVATE "TINYUSDZ_NO_STB_IMAGE_WRITE_IMPLEMENTATION")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
if(TINYUSDZ_WITH_ZSTD_COMPRESSION)
|
||||||
|
target_compile_definitions(${TINYUSDZ_LIB_TARGET} PRIVATE "TINYUSDZ_WITH_ZSTD_COMPRESSION")
|
||||||
|
if(TINYUSDZ_USE_SYSTEM_ZSTD)
|
||||||
|
target_link_libraries(${TINYUSDZ_LIB_TARGET} PUBLIC zstd::zstd)
|
||||||
|
endif()
|
||||||
|
endif()
|
||||||
|
|
||||||
target_include_directories(${TINYUSDZ_LIB_TARGET}
|
target_include_directories(${TINYUSDZ_LIB_TARGET}
|
||||||
PRIVATE ${PROJECT_SOURCE_DIR}/src)
|
PRIVATE ${PROJECT_SOURCE_DIR}/src)
|
||||||
|
|
||||||
|
|||||||
BIN
models/suzanne-pbr.usda.zst
Normal file
BIN
models/suzanne-pbr.usda.zst
Normal file
Binary file not shown.
51006
src/external/zstd.c
vendored
Normal file
51006
src/external/zstd.c
vendored
Normal file
File diff suppressed because it is too large
Load Diff
3089
src/external/zstd.h
vendored
Normal file
3089
src/external/zstd.h
vendored
Normal file
File diff suppressed because it is too large
Load Diff
@@ -26,6 +26,7 @@
|
|||||||
#include "integerCoding.h"
|
#include "integerCoding.h"
|
||||||
#include "io-util.hh"
|
#include "io-util.hh"
|
||||||
#include "lz4-compression.hh"
|
#include "lz4-compression.hh"
|
||||||
|
#include "zstd-compression.hh"
|
||||||
#include "pprinter.hh"
|
#include "pprinter.hh"
|
||||||
#include "str-util.hh"
|
#include "str-util.hh"
|
||||||
#include "stream-reader.hh"
|
#include "stream-reader.hh"
|
||||||
@@ -920,6 +921,51 @@ bool LoadUSDFromMemory(const uint8_t *addr, const size_t length,
|
|||||||
const std::string &base_dir, Stage *stage,
|
const std::string &base_dir, Stage *stage,
|
||||||
std::string *warn, std::string *err,
|
std::string *warn, std::string *err,
|
||||||
const USDLoadOptions &options) {
|
const USDLoadOptions &options) {
|
||||||
|
// Check for zstd-compressed data first (file-level compression)
|
||||||
|
if (IsZstdCompressed(addr, length)) {
|
||||||
|
DCOUT("Detected as zstd-compressed USD.");
|
||||||
|
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
// Get decompressed size for memory budget check
|
||||||
|
std::string zstd_err;
|
||||||
|
size_t decompressed_size = ZstdCompression::GetDecompressedSize(addr, length, &zstd_err);
|
||||||
|
if (decompressed_size == 0) {
|
||||||
|
if (err) {
|
||||||
|
(*err) += "Failed to get zstd decompressed size: " + zstd_err + "\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check against memory budget
|
||||||
|
size_t max_length = size_t(1024) * size_t(1024) * size_t(options.max_memory_limit_in_mb);
|
||||||
|
if (decompressed_size > max_length) {
|
||||||
|
if (err) {
|
||||||
|
(*err) += "Decompressed USD size (" + std::to_string(decompressed_size) +
|
||||||
|
" bytes) exceeds memory limit (" + std::to_string(max_length) + " bytes)\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decompress
|
||||||
|
std::vector<uint8_t> decompressed_data;
|
||||||
|
if (!ZstdCompression::Decompress(addr, length, &decompressed_data, &zstd_err)) {
|
||||||
|
if (err) {
|
||||||
|
(*err) += "Failed to decompress zstd data: " + zstd_err + "\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively call LoadUSDFromMemory with decompressed data
|
||||||
|
return LoadUSDFromMemory(decompressed_data.data(), decompressed_data.size(),
|
||||||
|
base_dir, stage, warn, err, options);
|
||||||
|
#else
|
||||||
|
if (err) {
|
||||||
|
(*err) += "zstd-compressed USD file detected, but zstd compression support is not enabled. "
|
||||||
|
"Rebuild with TINYUSDZ_WITH_ZSTD_COMPRESSION=ON.\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
if (IsUSDC(addr, length)) {
|
if (IsUSDC(addr, length)) {
|
||||||
DCOUT("Detected as USDC.");
|
DCOUT("Detected as USDC.");
|
||||||
return LoadUSDCFromMemory(addr, length, base_dir, stage, warn, err,
|
return LoadUSDCFromMemory(addr, length, base_dir, stage, warn, err,
|
||||||
@@ -1086,6 +1132,10 @@ bool IsUSDZ(const uint8_t *addr, const size_t length) {
|
|||||||
return ParseUSDZHeader(addr, length, /* [out] assets */ nullptr, &warn, &err);
|
return ParseUSDZHeader(addr, length, /* [out] assets */ nullptr, &warn, &err);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool IsZstdCompressed(const uint8_t *addr, const size_t length) {
|
||||||
|
return ZstdCompression::IsZstdCompressed(addr, length);
|
||||||
|
}
|
||||||
|
|
||||||
bool IsUSD(const std::string &filename, std::string *detected_format) {
|
bool IsUSD(const std::string &filename, std::string *detected_format) {
|
||||||
if (IsUSDA(filename)) {
|
if (IsUSDA(filename)) {
|
||||||
if (detected_format) {
|
if (detected_format) {
|
||||||
|
|||||||
@@ -138,13 +138,24 @@ struct USDLoadOptions {
|
|||||||
// - Realtime(moderate resource size limit)
|
// - Realtime(moderate resource size limit)
|
||||||
// - DCC(for data conversion. Unlimited resource size)
|
// - DCC(for data conversion. Unlimited resource size)
|
||||||
|
|
||||||
#if 0 // TODO
|
///
|
||||||
//struct USDWriteOptions
|
/// Options for writing USD files.
|
||||||
//{
|
///
|
||||||
//
|
struct USDWriteOptions {
|
||||||
//
|
///
|
||||||
//};
|
/// Enable zstd compression for output file.
|
||||||
#endif
|
/// When enabled, the entire USD file is wrapped with zstd compression.
|
||||||
|
/// Also auto-detected when filename ends with ".zst" extension.
|
||||||
|
///
|
||||||
|
bool use_zstd_compression{false};
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Zstd compression level (1-22).
|
||||||
|
/// Higher values give better compression but slower speed.
|
||||||
|
/// Default is 5 (good balance of speed and ratio).
|
||||||
|
///
|
||||||
|
int zstd_compression_level{5};
|
||||||
|
};
|
||||||
|
|
||||||
//
|
//
|
||||||
|
|
||||||
@@ -469,6 +480,10 @@ bool IsUSDC(const uint8_t *addr, const size_t length);
|
|||||||
bool IsUSDZ(const std::string &filename);
|
bool IsUSDZ(const std::string &filename);
|
||||||
bool IsUSDZ(const uint8_t *addr, const size_t length);
|
bool IsUSDZ(const uint8_t *addr, const size_t length);
|
||||||
|
|
||||||
|
// Test if input is zstd-compressed data (by magic number).
|
||||||
|
// This is for file-level compression, not internal USDC LZ4 compression.
|
||||||
|
bool IsZstdCompressed(const uint8_t *addr, const size_t length);
|
||||||
|
|
||||||
} // namespace tinyusdz
|
} // namespace tinyusdz
|
||||||
|
|
||||||
#endif // TINYUSDZ_HH_
|
#endif // TINYUSDZ_HH_
|
||||||
|
|||||||
@@ -17,46 +17,131 @@
|
|||||||
#include "value-pprint.hh"
|
#include "value-pprint.hh"
|
||||||
#include "tinyusdz.hh"
|
#include "tinyusdz.hh"
|
||||||
#include "io-util.hh"
|
#include "io-util.hh"
|
||||||
|
#include "str-util.hh"
|
||||||
|
#include "zstd-compression.hh"
|
||||||
|
|
||||||
namespace tinyusdz {
|
namespace tinyusdz {
|
||||||
namespace usda {
|
namespace usda {
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
|
// Check if filename ends with ".zst" extension (case-insensitive)
|
||||||
|
bool HasZstdExtension(const std::string &filename) {
|
||||||
|
if (filename.size() < 4) return false;
|
||||||
|
std::string ext = filename.substr(filename.size() - 4);
|
||||||
|
return (ext == ".zst" || ext == ".ZST");
|
||||||
|
}
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
bool HasZstdExtension(const std::wstring &filename) {
|
||||||
|
if (filename.size() < 4) return false;
|
||||||
|
std::wstring ext = filename.substr(filename.size() - 4);
|
||||||
|
return (ext == L".zst" || ext == L".ZST");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage,
|
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err) {
|
||||||
std::string *warn, std::string *err) {
|
|
||||||
|
|
||||||
(void)warn;
|
(void)warn;
|
||||||
|
(void)err;
|
||||||
|
|
||||||
// TODO: Handle warn and err on export.
|
if (!output) {
|
||||||
std::string s = stage.ExportToString();
|
if (err) {
|
||||||
|
(*err) = "output parameter is null.\n";
|
||||||
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::cout << "Wrote to [" << filename << "]\n";
|
*output = stage.ExportToString();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool SaveAsUSDA(const std::string &filename, const Stage &stage,
|
||||||
|
std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options) {
|
||||||
|
|
||||||
|
(void)warn;
|
||||||
|
|
||||||
|
// Export stage to string
|
||||||
|
std::string s = stage.ExportToString();
|
||||||
|
|
||||||
|
// Check if we should use zstd compression
|
||||||
|
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
|
||||||
|
|
||||||
|
if (use_compression) {
|
||||||
|
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
// Compress the data
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
if (!ZstdCompression::Compress(reinterpret_cast<const uint8_t*>(s.data()), s.size(),
|
||||||
|
&compressed, options.zstd_compression_level, err)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!io::WriteWholeFile(filename, compressed.data(), compressed.size(), err)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Wrote zstd-compressed USDA to [" << filename << "] ("
|
||||||
|
<< s.size() << " -> " << compressed.size() << " bytes)\n";
|
||||||
|
#else
|
||||||
|
if (err) {
|
||||||
|
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
#endif
|
||||||
|
} else {
|
||||||
|
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Wrote USDA to [" << filename << "]\n";
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage,
|
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage,
|
||||||
std::string *warn, std::string *err) {
|
std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options) {
|
||||||
|
|
||||||
(void)warn;
|
(void)warn;
|
||||||
|
|
||||||
// TODO: Handle warn and err on export.
|
// Export stage to string
|
||||||
std::string s = stage.ExportToString();
|
std::string s = stage.ExportToString();
|
||||||
|
|
||||||
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
// Check if we should use zstd compression
|
||||||
return false;
|
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
|
||||||
}
|
|
||||||
|
|
||||||
std::wcout << "Wrote to [" << filename << "]\n";
|
if (use_compression) {
|
||||||
|
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
// Compress the data
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
if (!ZstdCompression::Compress(reinterpret_cast<const uint8_t*>(s.data()), s.size(),
|
||||||
|
&compressed, options.zstd_compression_level, err)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!io::WriteWholeFile(filename, compressed.data(), compressed.size(), err)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::wcout << L"Wrote zstd-compressed USDA to [" << filename << L"] ("
|
||||||
|
<< s.size() << L" -> " << compressed.size() << L" bytes)\n";
|
||||||
|
#else
|
||||||
|
if (err) {
|
||||||
|
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
#endif
|
||||||
|
} else {
|
||||||
|
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::wcout << L"Wrote USDA to [" << filename << L"]\n";
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -71,9 +156,9 @@ bool SaveAsUSDA(const std::wstring &filename, const Stage &stage,
|
|||||||
namespace tinyusdz {
|
namespace tinyusdz {
|
||||||
namespace usda {
|
namespace usda {
|
||||||
|
|
||||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err) {
|
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err) {
|
||||||
(void)filename;
|
|
||||||
(void)stage;
|
(void)stage;
|
||||||
|
(void)output;
|
||||||
(void)warn;
|
(void)warn;
|
||||||
|
|
||||||
if (err) {
|
if (err) {
|
||||||
@@ -82,7 +167,33 @@ bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *wa
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options) {
|
||||||
|
(void)filename;
|
||||||
|
(void)stage;
|
||||||
|
(void)warn;
|
||||||
|
(void)options;
|
||||||
|
|
||||||
|
if (err) {
|
||||||
|
(*err) = "USDA Writer feature is disabled in this build.\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options) {
|
||||||
|
(void)filename;
|
||||||
|
(void)stage;
|
||||||
|
(void)warn;
|
||||||
|
(void)options;
|
||||||
|
|
||||||
|
if (err) {
|
||||||
|
(*err) = "USDA Writer feature is disabled in this build.\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
} // namespace usda
|
} // namespace usda
|
||||||
} // namespace tinyusdz
|
} // namespace tinyusdz
|
||||||
|
|||||||
@@ -16,18 +16,34 @@ namespace usda {
|
|||||||
/// Save scene as USDA(ASCII)
|
/// Save scene as USDA(ASCII)
|
||||||
///
|
///
|
||||||
/// @param[in] filename USDA filename(UTF-8). WideChar(Unicode) represented as std::string is supported on Windows.
|
/// @param[in] filename USDA filename(UTF-8). WideChar(Unicode) represented as std::string is supported on Windows.
|
||||||
|
/// If filename ends with ".zst", zstd compression is automatically enabled.
|
||||||
/// @param[in] stage Stage(scene graph).
|
/// @param[in] stage Stage(scene graph).
|
||||||
/// @param[out] warn Warning message
|
/// @param[out] warn Warning message
|
||||||
/// @param[out] err Error message
|
/// @param[out] err Error message
|
||||||
|
/// @param[in] options Write options (optional). Includes zstd compression settings.
|
||||||
|
///
|
||||||
|
/// @return true upon success.
|
||||||
|
///
|
||||||
|
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options = USDWriteOptions());
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
// WideChar(UNICODE) filename version.
|
||||||
|
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options = USDWriteOptions());
|
||||||
|
#endif
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Export stage as USDA string.
|
||||||
|
///
|
||||||
|
/// @param[in] stage Stage(scene graph).
|
||||||
|
/// @param[out] output USDA string output
|
||||||
|
/// @param[out] warn Warning message
|
||||||
|
/// @param[out] err Error message
|
||||||
///
|
///
|
||||||
/// @return true upon success.
|
/// @return true upon success.
|
||||||
///
|
///
|
||||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err);
|
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err);
|
||||||
|
|
||||||
#if defined(_WIN32)
|
|
||||||
// WideChar(UNICODE) filename version.
|
|
||||||
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
} // namespace usda
|
} // namespace usda
|
||||||
} // namespace tinyusdz
|
} // namespace tinyusdz
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
#include "crate-format.hh"
|
#include "crate-format.hh"
|
||||||
#include "io-util.hh"
|
#include "io-util.hh"
|
||||||
#include "lz4-compression.hh"
|
#include "lz4-compression.hh"
|
||||||
|
#include "zstd-compression.hh"
|
||||||
#include "token-type.hh"
|
#include "token-type.hh"
|
||||||
|
|
||||||
#include "common-macros.inc"
|
#include "common-macros.inc"
|
||||||
@@ -50,6 +51,13 @@ namespace usdc {
|
|||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
|
// Check if filename ends with ".zst" extension (case-insensitive)
|
||||||
|
bool HasZstdExtension(const std::string &filename) {
|
||||||
|
if (filename.size() < 4) return false;
|
||||||
|
std::string ext = filename.substr(filename.size() - 4);
|
||||||
|
return (ext == ".zst" || ext == ".ZST");
|
||||||
|
}
|
||||||
|
|
||||||
constexpr size_t kSectionNameMaxLength = 15;
|
constexpr size_t kSectionNameMaxLength = 15;
|
||||||
|
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
@@ -480,11 +488,13 @@ class Writer {
|
|||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||||
std::string *warn, std::string *err) {
|
std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options) {
|
||||||
#ifdef __ANDROID__
|
#ifdef __ANDROID__
|
||||||
(void)filename;
|
(void)filename;
|
||||||
(void)stage;
|
(void)stage;
|
||||||
(void)warn;
|
(void)warn;
|
||||||
|
(void)options;
|
||||||
|
|
||||||
if (err) {
|
if (err) {
|
||||||
(*err) += "Saving USDC to a file is not supported for Android platform(at the moment).\n";
|
(*err) += "Saving USDC to a file is not supported for Android platform(at the moment).\n";
|
||||||
@@ -498,6 +508,30 @@ bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if we should use zstd compression
|
||||||
|
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
|
||||||
|
|
||||||
|
const uint8_t *write_data = output.data();
|
||||||
|
size_t write_size = output.size();
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
|
||||||
|
if (use_compression) {
|
||||||
|
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
if (!ZstdCompression::Compress(output.data(), output.size(),
|
||||||
|
&compressed, options.zstd_compression_level, err)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
write_data = compressed.data();
|
||||||
|
write_size = compressed.size();
|
||||||
|
std::cout << "Compressing USDC with zstd (" << output.size() << " -> " << compressed.size() << " bytes)\n";
|
||||||
|
#else
|
||||||
|
if (err) {
|
||||||
|
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
#if defined(_MSC_VER) || defined(__GLIBCXX__) || defined(__clang__)
|
#if defined(_MSC_VER) || defined(__GLIBCXX__) || defined(__clang__)
|
||||||
FILE *fp = nullptr;
|
FILE *fp = nullptr;
|
||||||
@@ -530,9 +564,11 @@ bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
size_t n = fwrite(output.data(), /* size */ 1, /* count */ output.size(), fp);
|
size_t n = fwrite(write_data, /* size */ 1, /* count */ write_size, fp);
|
||||||
if (n < output.size()) {
|
fclose(fp);
|
||||||
// TODO: Retry writing data when n < output.size()
|
|
||||||
|
if (n < write_size) {
|
||||||
|
// TODO: Retry writing data when n < write_size
|
||||||
|
|
||||||
if (err) {
|
if (err) {
|
||||||
(*err) += "Failed to write data to a file.\n";
|
(*err) += "Failed to write data to a file.\n";
|
||||||
@@ -568,10 +604,12 @@ namespace tinyusdz {
|
|||||||
namespace usdc {
|
namespace usdc {
|
||||||
|
|
||||||
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||||
std::string *warn, std::string *err) {
|
std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options) {
|
||||||
(void)filename;
|
(void)filename;
|
||||||
(void)stage;
|
(void)stage;
|
||||||
(void)warn;
|
(void)warn;
|
||||||
|
(void)options;
|
||||||
|
|
||||||
if (err) {
|
if (err) {
|
||||||
(*err) = "USDC writer feature is disabled in this build.\n";
|
(*err) = "USDC writer feature is disabled in this build.\n";
|
||||||
|
|||||||
@@ -19,15 +19,17 @@ namespace usdc {
|
|||||||
///
|
///
|
||||||
/// Save scene as USDC(binary) to a file
|
/// Save scene as USDC(binary) to a file
|
||||||
///
|
///
|
||||||
/// @param[in] filename USDC filename
|
/// @param[in] filename USDC filename. If filename ends with ".zst", zstd compression is auto-enabled.
|
||||||
/// @param[in] stage Stage
|
/// @param[in] stage Stage
|
||||||
/// @param[out] warn Warning message
|
/// @param[out] warn Warning message
|
||||||
/// @param[out] err Error message
|
/// @param[out] err Error message
|
||||||
|
/// @param[in] options Write options (optional). Includes zstd compression settings.
|
||||||
///
|
///
|
||||||
/// @return true upon success.
|
/// @return true upon success.
|
||||||
///
|
///
|
||||||
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||||
std::string *warn, std::string *err);
|
std::string *warn, std::string *err,
|
||||||
|
const USDWriteOptions &options = USDWriteOptions());
|
||||||
|
|
||||||
///
|
///
|
||||||
/// Save scene as USDC(binary) to a memory
|
/// Save scene as USDC(binary) to a memory
|
||||||
|
|||||||
175
src/zstd-compression.cc
Normal file
175
src/zstd-compression.cc
Normal file
@@ -0,0 +1,175 @@
|
|||||||
|
// SPDX-License-Identifier: Apache 2.0
|
||||||
|
// Copyright 2024-Present Light Transport Entertainment Inc.
|
||||||
|
//
|
||||||
|
// Zstd compression wrapper implementation for TinyUSDZ
|
||||||
|
|
||||||
|
#include "zstd-compression.hh"
|
||||||
|
|
||||||
|
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
|
||||||
|
#include <cstring>
|
||||||
|
#include <limits>
|
||||||
|
|
||||||
|
// Use amalgamated single-file zstd
|
||||||
|
#include "external/zstd.h"
|
||||||
|
|
||||||
|
namespace tinyusdz {
|
||||||
|
|
||||||
|
bool ZstdCompression::IsZstdCompressed(const uint8_t *data, size_t length) {
|
||||||
|
if (!data || length < 4) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (data[0] == kZstdMagic[0] && data[1] == kZstdMagic[1] &&
|
||||||
|
data[2] == kZstdMagic[2] && data[3] == kZstdMagic[3]);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t ZstdCompression::GetDecompressedSize(const uint8_t *compressed,
|
||||||
|
size_t compressedSize,
|
||||||
|
std::string *err) {
|
||||||
|
if (!compressed || compressedSize == 0) {
|
||||||
|
if (err) *err = "Invalid compressed data";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long frameContentSize =
|
||||||
|
ZSTD_getFrameContentSize(compressed, compressedSize);
|
||||||
|
|
||||||
|
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR) {
|
||||||
|
if (err) *err = "Not a valid zstd compressed frame";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
|
||||||
|
if (err) *err = "Zstd frame does not contain content size";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for size overflow
|
||||||
|
if (frameContentSize > std::numeric_limits<size_t>::max()) {
|
||||||
|
if (err) *err = "Decompressed size exceeds size_t range";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return static_cast<size_t>(frameContentSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ZstdCompression::Decompress(const uint8_t *compressed,
|
||||||
|
size_t compressedSize,
|
||||||
|
std::vector<uint8_t> *output,
|
||||||
|
std::string *err) {
|
||||||
|
if (!compressed || compressedSize == 0) {
|
||||||
|
if (err) *err = "Invalid compressed data";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!output) {
|
||||||
|
if (err) *err = "Output buffer is null";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get decompressed size first
|
||||||
|
size_t decompressedSize = GetDecompressedSize(compressed, compressedSize, err);
|
||||||
|
if (decompressedSize == 0) {
|
||||||
|
// err already set by GetDecompressedSize
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allocate output buffer
|
||||||
|
// Note: TinyUSDZ has exceptions disabled, so we check capacity instead
|
||||||
|
output->clear();
|
||||||
|
if (decompressedSize > output->max_size()) {
|
||||||
|
if (err) *err = "Decompressed size exceeds maximum allocatable size";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
output->resize(decompressedSize);
|
||||||
|
if (output->size() != decompressedSize) {
|
||||||
|
if (err) *err = "Failed to allocate memory for decompression";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decompress
|
||||||
|
size_t result =
|
||||||
|
ZSTD_decompress(output->data(), decompressedSize, compressed, compressedSize);
|
||||||
|
|
||||||
|
if (ZSTD_isError(result)) {
|
||||||
|
if (err) {
|
||||||
|
*err = "Zstd decompression failed: ";
|
||||||
|
*err += ZSTD_getErrorName(result);
|
||||||
|
}
|
||||||
|
output->clear();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify decompressed size matches expected
|
||||||
|
if (result != decompressedSize) {
|
||||||
|
if (err) {
|
||||||
|
*err = "Decompressed size mismatch: expected " +
|
||||||
|
std::to_string(decompressedSize) + ", got " + std::to_string(result);
|
||||||
|
}
|
||||||
|
output->clear();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ZstdCompression::Compress(const uint8_t *input, size_t inputSize,
|
||||||
|
std::vector<uint8_t> *output,
|
||||||
|
int compressionLevel, std::string *err) {
|
||||||
|
if (!input || inputSize == 0) {
|
||||||
|
if (err) *err = "Invalid input data";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!output) {
|
||||||
|
if (err) *err = "Output buffer is null";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clamp compression level to valid range
|
||||||
|
if (compressionLevel < 1) compressionLevel = 1;
|
||||||
|
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
|
||||||
|
|
||||||
|
// Get maximum compressed size
|
||||||
|
size_t maxCompressedSize = ZSTD_compressBound(inputSize);
|
||||||
|
|
||||||
|
// Allocate output buffer
|
||||||
|
// Note: TinyUSDZ has exceptions disabled, so we check capacity instead
|
||||||
|
output->clear();
|
||||||
|
if (maxCompressedSize > output->max_size()) {
|
||||||
|
if (err) *err = "Compressed buffer size exceeds maximum allocatable size";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
output->resize(maxCompressedSize);
|
||||||
|
if (output->size() != maxCompressedSize) {
|
||||||
|
if (err) *err = "Failed to allocate memory for compression";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compress
|
||||||
|
size_t result =
|
||||||
|
ZSTD_compress(output->data(), maxCompressedSize, input, inputSize,
|
||||||
|
compressionLevel);
|
||||||
|
|
||||||
|
if (ZSTD_isError(result)) {
|
||||||
|
if (err) {
|
||||||
|
*err = "Zstd compression failed: ";
|
||||||
|
*err += ZSTD_getErrorName(result);
|
||||||
|
}
|
||||||
|
output->clear();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shrink to actual compressed size
|
||||||
|
output->resize(result);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t ZstdCompression::GetCompressBound(size_t inputSize) {
|
||||||
|
return ZSTD_compressBound(inputSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace tinyusdz
|
||||||
|
|
||||||
|
#endif // TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
116
src/zstd-compression.hh
Normal file
116
src/zstd-compression.hh
Normal file
@@ -0,0 +1,116 @@
|
|||||||
|
// SPDX-License-Identifier: Apache 2.0
|
||||||
|
// Copyright 2024-Present Light Transport Entertainment Inc.
|
||||||
|
//
|
||||||
|
// Zstd compression wrapper for TinyUSDZ
|
||||||
|
// Provides file-level zstd compression/decompression for USD files
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#ifndef ZSTD_COMPRESSION_HH_
|
||||||
|
#define ZSTD_COMPRESSION_HH_
|
||||||
|
|
||||||
|
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace tinyusdz {
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Zstd compression utility class.
|
||||||
|
/// Provides static methods for compressing and decompressing data using zstd.
|
||||||
|
///
|
||||||
|
class ZstdCompression {
|
||||||
|
public:
|
||||||
|
///
|
||||||
|
/// Check if data starts with zstd magic number (0x28 0xB5 0x2F 0xFD)
|
||||||
|
/// @param[in] data Pointer to the data to check
|
||||||
|
/// @param[in] length Length of the data in bytes
|
||||||
|
/// @return true if data starts with zstd magic number
|
||||||
|
///
|
||||||
|
static bool IsZstdCompressed(const uint8_t *data, size_t length);
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Get the decompressed size of zstd-compressed data.
|
||||||
|
/// This can be used to check memory budget before decompression.
|
||||||
|
/// @param[in] compressed Pointer to compressed data
|
||||||
|
/// @param[in] compressedSize Size of compressed data in bytes
|
||||||
|
/// @param[out] err Error message if size cannot be determined
|
||||||
|
/// @return Decompressed size in bytes, or 0 on error
|
||||||
|
///
|
||||||
|
static size_t GetDecompressedSize(const uint8_t *compressed,
|
||||||
|
size_t compressedSize, std::string *err);
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Decompress zstd-compressed data.
|
||||||
|
/// @param[in] compressed Pointer to compressed data
|
||||||
|
/// @param[in] compressedSize Size of compressed data in bytes
|
||||||
|
/// @param[out] output Vector to store decompressed data
|
||||||
|
/// @param[out] err Error message on failure
|
||||||
|
/// @return true on success, false on failure
|
||||||
|
///
|
||||||
|
static bool Decompress(const uint8_t *compressed, size_t compressedSize,
|
||||||
|
std::vector<uint8_t> *output, std::string *err);
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Compress data using zstd.
|
||||||
|
/// @param[in] input Pointer to input data
|
||||||
|
/// @param[in] inputSize Size of input data in bytes
|
||||||
|
/// @param[out] output Vector to store compressed data
|
||||||
|
/// @param[in] compressionLevel Compression level (1-22, default 5)
|
||||||
|
/// @param[out] err Error message on failure
|
||||||
|
/// @return true on success, false on failure
|
||||||
|
///
|
||||||
|
static bool Compress(const uint8_t *input, size_t inputSize,
|
||||||
|
std::vector<uint8_t> *output, int compressionLevel,
|
||||||
|
std::string *err);
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Get the maximum compressed size for a given input size.
|
||||||
|
/// @param[in] inputSize Size of input data in bytes
|
||||||
|
/// @return Maximum possible compressed size
|
||||||
|
///
|
||||||
|
static size_t GetCompressBound(size_t inputSize);
|
||||||
|
|
||||||
|
/// Default compression level (5 is a good balance of speed and ratio)
|
||||||
|
static constexpr int kDefaultCompressionLevel = 5;
|
||||||
|
|
||||||
|
/// Zstd magic number bytes
|
||||||
|
static constexpr uint8_t kZstdMagic[4] = {0x28, 0xB5, 0x2F, 0xFD};
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace tinyusdz
|
||||||
|
|
||||||
|
#else // !TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
|
||||||
|
// Stub implementation when zstd compression is disabled
|
||||||
|
namespace tinyusdz {
|
||||||
|
|
||||||
|
class ZstdCompression {
|
||||||
|
public:
|
||||||
|
static bool IsZstdCompressed(const uint8_t *, size_t) { return false; }
|
||||||
|
static size_t GetDecompressedSize(const uint8_t *, size_t, std::string *err) {
|
||||||
|
if (err) *err = "Zstd compression support not enabled";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
static bool Decompress(const uint8_t *, size_t, std::vector<uint8_t> *,
|
||||||
|
std::string *err) {
|
||||||
|
if (err) *err = "Zstd compression support not enabled";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
static bool Compress(const uint8_t *, size_t, std::vector<uint8_t> *, int,
|
||||||
|
std::string *err) {
|
||||||
|
if (err) *err = "Zstd compression support not enabled";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
static size_t GetCompressBound(size_t) { return 0; }
|
||||||
|
static constexpr int kDefaultCompressionLevel = 5;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace tinyusdz
|
||||||
|
|
||||||
|
#endif // TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
|
||||||
|
#endif // ZSTD_COMPRESSION_HH_
|
||||||
76
tests/feat/zstdusd/Makefile
Normal file
76
tests/feat/zstdusd/Makefile
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
# Makefile for zstd USD compression feature tests
|
||||||
|
# This is a standalone Makefile that builds and runs the zstd compression tests
|
||||||
|
|
||||||
|
CXX = g++
|
||||||
|
CXXFLAGS = -std=c++14 -O2 -g -Wall -Wextra
|
||||||
|
|
||||||
|
# TinyUSDZ source directory
|
||||||
|
TINYUSDZ_DIR = ../../..
|
||||||
|
SRC_DIR = $(TINYUSDZ_DIR)/src
|
||||||
|
EXT_DIR = $(SRC_DIR)/external
|
||||||
|
|
||||||
|
# Include paths
|
||||||
|
INCLUDES = -I$(SRC_DIR) -I$(EXT_DIR)
|
||||||
|
|
||||||
|
# Define to enable zstd compression
|
||||||
|
DEFINES = -DTINYUSDZ_WITH_ZSTD_COMPRESSION -DZSTD_DISABLE_ASM=1
|
||||||
|
|
||||||
|
# Zstd source (amalgamated single file)
|
||||||
|
ZSTD_SRC = $(EXT_DIR)/zstd.c
|
||||||
|
|
||||||
|
# TinyUSDZ sources needed for this test
|
||||||
|
TINYUSDZ_SRCS = \
|
||||||
|
$(SRC_DIR)/zstd-compression.cc \
|
||||||
|
$(SRC_DIR)/tinyusdz.cc \
|
||||||
|
$(SRC_DIR)/usda-writer.cc \
|
||||||
|
$(SRC_DIR)/usda-reader.cc \
|
||||||
|
$(SRC_DIR)/usdc-reader.cc \
|
||||||
|
$(SRC_DIR)/crate-reader.cc \
|
||||||
|
$(SRC_DIR)/ascii-parser.cc \
|
||||||
|
$(SRC_DIR)/prim-types.cc \
|
||||||
|
$(SRC_DIR)/value-types.cc \
|
||||||
|
$(SRC_DIR)/pprinter.cc \
|
||||||
|
$(SRC_DIR)/value-pprint.cc \
|
||||||
|
$(SRC_DIR)/io-util.cc \
|
||||||
|
$(SRC_DIR)/str-util.cc \
|
||||||
|
$(SRC_DIR)/primvar.cc \
|
||||||
|
$(SRC_DIR)/stage.cc \
|
||||||
|
$(SRC_DIR)/prim-composition.cc \
|
||||||
|
$(SRC_DIR)/composition.cc \
|
||||||
|
$(SRC_DIR)/asset-resolution.cc \
|
||||||
|
$(SRC_DIR)/path-util.cc \
|
||||||
|
$(SRC_DIR)/image-loader.cc \
|
||||||
|
$(SRC_DIR)/usdGeom.cc \
|
||||||
|
$(SRC_DIR)/usdSkel.cc \
|
||||||
|
$(SRC_DIR)/usdShade.cc \
|
||||||
|
$(SRC_DIR)/usdLux.cc \
|
||||||
|
$(SRC_DIR)/usdRender.cc \
|
||||||
|
$(SRC_DIR)/layer.cc \
|
||||||
|
$(SRC_DIR)/lz4-compression.cc \
|
||||||
|
$(EXT_DIR)/lz4/lz4.c \
|
||||||
|
$(EXT_DIR)/fpng.cpp \
|
||||||
|
$(EXT_DIR)/string_id/database.cpp \
|
||||||
|
$(EXT_DIR)/string_id/error.cpp \
|
||||||
|
$(EXT_DIR)/string_id/string_id.cpp \
|
||||||
|
$(EXT_DIR)/ryu/s2d.c \
|
||||||
|
$(EXT_DIR)/ryu/d2s.c \
|
||||||
|
$(EXT_DIR)/tsl/robin_hood.h
|
||||||
|
|
||||||
|
# Test source
|
||||||
|
TEST_SRC = test_zstd_usd.cc
|
||||||
|
|
||||||
|
# Output binary
|
||||||
|
TARGET = test_zstd_usd
|
||||||
|
|
||||||
|
.PHONY: all clean test
|
||||||
|
|
||||||
|
all: $(TARGET)
|
||||||
|
|
||||||
|
$(TARGET): $(TEST_SRC) $(TINYUSDZ_SRCS) $(ZSTD_SRC)
|
||||||
|
$(CXX) $(CXXFLAGS) $(INCLUDES) $(DEFINES) -o $@ $(TEST_SRC) $(TINYUSDZ_SRCS) $(ZSTD_SRC) -lpthread
|
||||||
|
|
||||||
|
test: $(TARGET)
|
||||||
|
./$(TARGET)
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -f $(TARGET) test_output.usda.zst
|
||||||
51
tests/feat/zstdusd/README.md
Normal file
51
tests/feat/zstdusd/README.md
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
# Zstd USD Compression Feature Tests
|
||||||
|
|
||||||
|
This directory contains feature tests for zstd-compressed USD file support in TinyUSDZ.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
TinyUSDZ supports file-level zstd compression for USD files. This allows:
|
||||||
|
- Reading zstd-compressed USDA/USDC/USDZ files (automatic detection)
|
||||||
|
- Writing zstd-compressed USD files (via `USDWriteOptions` or `.zst` extension)
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make test
|
||||||
|
```
|
||||||
|
|
||||||
|
Or directly:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./test_zstd_usd
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test Cases
|
||||||
|
|
||||||
|
1. **Magic Number Detection** - Verifies zstd magic number detection
|
||||||
|
2. **Compression Round-trip** - Basic compress/decompress cycle
|
||||||
|
3. **Compression Levels** - Tests different compression levels (1-22)
|
||||||
|
4. **Corrupt Data Handling** - Error handling for invalid data
|
||||||
|
5. **GetCompressBound** - Tests the compress bound calculation
|
||||||
|
6. **USDA Round-trip** - Full USDA compression/decompression cycle
|
||||||
|
7. **IsZstdCompressed Wrapper** - Tests the tinyusdz namespace wrapper
|
||||||
|
8. **Load Compressed USDA from Memory** - Tests LoadUSDFromMemory with zstd data
|
||||||
|
9. **Memory Budget Enforcement** - Tests memory limit checking
|
||||||
|
10. **Write Compressed USDA** - Tests SaveAsUSDA with compression
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
- C++14 compiler
|
||||||
|
- TinyUSDZ source tree
|
||||||
|
- Zstd library (bundled in src/external/zstd.c)
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- The tests use the amalgamated single-file zstd library
|
||||||
|
- ZSTD_DISABLE_ASM=1 is set to avoid assembly-related linking issues
|
||||||
130
tests/feat/zstdusd/compress_usda.cc
Normal file
130
tests/feat/zstdusd/compress_usda.cc
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
// Simple utility to compress a USDA file with zstd
|
||||||
|
// Build: g++ -std=c++14 -o compress_usda compress_usda.cc -I../../../src -I../../../src/external ../../../src/external/zstd.c -DTINYUSDZ_WITH_ZSTD_COMPRESSION -DZSTD_DISABLE_ASM=1 -w
|
||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
#include <vector>
|
||||||
|
#include <cstring>
|
||||||
|
|
||||||
|
#define TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
#include "zstd-compression.hh"
|
||||||
|
|
||||||
|
// Include zstd header for the implementation
|
||||||
|
#include "external/zstd.h"
|
||||||
|
|
||||||
|
// Implement the static members
|
||||||
|
namespace tinyusdz {
|
||||||
|
|
||||||
|
constexpr uint8_t ZstdCompression::kZstdMagic[4];
|
||||||
|
|
||||||
|
bool ZstdCompression::IsZstdCompressed(const uint8_t *data, size_t length) {
|
||||||
|
if (!data || length < 4) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (data[0] == kZstdMagic[0] && data[1] == kZstdMagic[1] &&
|
||||||
|
data[2] == kZstdMagic[2] && data[3] == kZstdMagic[3]);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ZstdCompression::Compress(const uint8_t *input, size_t inputSize,
|
||||||
|
std::vector<uint8_t> *output,
|
||||||
|
int compressionLevel, std::string *err) {
|
||||||
|
if (!input || inputSize == 0) {
|
||||||
|
if (err) *err = "Invalid input data";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!output) {
|
||||||
|
if (err) *err = "Output buffer is null";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clamp compression level to valid range
|
||||||
|
if (compressionLevel < 1) compressionLevel = 1;
|
||||||
|
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
|
||||||
|
|
||||||
|
// Get maximum compressed size
|
||||||
|
size_t maxCompressedSize = ZSTD_compressBound(inputSize);
|
||||||
|
|
||||||
|
// Allocate output buffer
|
||||||
|
output->resize(maxCompressedSize);
|
||||||
|
|
||||||
|
// Compress
|
||||||
|
size_t result =
|
||||||
|
ZSTD_compress(output->data(), maxCompressedSize, input, inputSize,
|
||||||
|
compressionLevel);
|
||||||
|
|
||||||
|
if (ZSTD_isError(result)) {
|
||||||
|
if (err) {
|
||||||
|
*err = "Zstd compression failed: ";
|
||||||
|
*err += ZSTD_getErrorName(result);
|
||||||
|
}
|
||||||
|
output->clear();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shrink to actual compressed size
|
||||||
|
output->resize(result);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t ZstdCompression::GetCompressBound(size_t inputSize) {
|
||||||
|
return ZSTD_compressBound(inputSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace tinyusdz
|
||||||
|
|
||||||
|
int main(int argc, char** argv) {
|
||||||
|
if (argc < 3) {
|
||||||
|
std::cerr << "Usage: " << argv[0] << " <input.usda> <output.usda.zst> [compression_level]" << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* input_file = argv[1];
|
||||||
|
const char* output_file = argv[2];
|
||||||
|
int compression_level = 5;
|
||||||
|
if (argc >= 4) {
|
||||||
|
compression_level = std::atoi(argv[3]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read input file
|
||||||
|
std::ifstream ifs(input_file, std::ios::binary);
|
||||||
|
if (!ifs.is_open()) {
|
||||||
|
std::cerr << "Failed to open input file: " << input_file << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<uint8_t> input_data((std::istreambuf_iterator<char>(ifs)),
|
||||||
|
std::istreambuf_iterator<char>());
|
||||||
|
ifs.close();
|
||||||
|
|
||||||
|
std::cout << "Input file: " << input_file << std::endl;
|
||||||
|
std::cout << "Input size: " << input_data.size() << " bytes" << std::endl;
|
||||||
|
|
||||||
|
// Compress
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
std::string err;
|
||||||
|
bool ok = tinyusdz::ZstdCompression::Compress(
|
||||||
|
input_data.data(), input_data.size(), &compressed, compression_level, &err);
|
||||||
|
|
||||||
|
if (!ok) {
|
||||||
|
std::cerr << "Compression failed: " << err << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Compressed size: " << compressed.size() << " bytes" << std::endl;
|
||||||
|
std::cout << "Compression ratio: " << (100.0 * compressed.size() / input_data.size()) << "%" << std::endl;
|
||||||
|
|
||||||
|
// Write output file
|
||||||
|
std::ofstream ofs(output_file, std::ios::binary);
|
||||||
|
if (!ofs.is_open()) {
|
||||||
|
std::cerr << "Failed to open output file: " << output_file << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
ofs.write(reinterpret_cast<const char*>(compressed.data()), compressed.size());
|
||||||
|
ofs.close();
|
||||||
|
|
||||||
|
std::cout << "Wrote compressed file: " << output_file << std::endl;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
388
tests/feat/zstdusd/test_zstd_usd.cc
Normal file
388
tests/feat/zstdusd/test_zstd_usd.cc
Normal file
@@ -0,0 +1,388 @@
|
|||||||
|
// SPDX-License-Identifier: Apache 2.0
|
||||||
|
// Copyright 2024-Present Light Transport Entertainment Inc.
|
||||||
|
//
|
||||||
|
// Feature test for zstd-compressed USD file support
|
||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
#include <cstring>
|
||||||
|
#include <cstdlib>
|
||||||
|
|
||||||
|
#define TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||||
|
|
||||||
|
#include "tinyusdz.hh"
|
||||||
|
#include "usda-writer.hh"
|
||||||
|
#include "zstd-compression.hh"
|
||||||
|
|
||||||
|
// Simple test macros
|
||||||
|
#define TEST_ASSERT(cond, msg) do { \
|
||||||
|
if (!(cond)) { \
|
||||||
|
std::cerr << "FAILED: " << msg << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
|
||||||
|
return 1; \
|
||||||
|
} \
|
||||||
|
std::cout << "PASSED: " << msg << std::endl; \
|
||||||
|
} while(0)
|
||||||
|
|
||||||
|
#define TEST_ASSERT_FALSE(cond, msg) TEST_ASSERT(!(cond), msg)
|
||||||
|
|
||||||
|
// Test 1: Magic number detection
|
||||||
|
int test_magic_number_detection() {
|
||||||
|
std::cout << "\n=== Test: Magic Number Detection ===" << std::endl;
|
||||||
|
|
||||||
|
// Valid zstd magic number
|
||||||
|
uint8_t valid_magic[] = {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x00, 0x00};
|
||||||
|
TEST_ASSERT(tinyusdz::ZstdCompression::IsZstdCompressed(valid_magic, 8),
|
||||||
|
"Should detect valid zstd magic");
|
||||||
|
|
||||||
|
// Invalid data
|
||||||
|
uint8_t invalid_data[] = {0x00, 0x00, 0x00, 0x00};
|
||||||
|
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(invalid_data, 4),
|
||||||
|
"Should reject non-zstd data");
|
||||||
|
|
||||||
|
// USDA header
|
||||||
|
uint8_t usda_header[] = {'#', 'u', 's', 'd', 'a', ' ', '1', '.'};
|
||||||
|
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(usda_header, 8),
|
||||||
|
"Should reject USDA header");
|
||||||
|
|
||||||
|
// USDC header
|
||||||
|
uint8_t usdc_header[] = {'P', 'X', 'R', '-', 'U', 'S', 'D', 'C'};
|
||||||
|
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(usdc_header, 8),
|
||||||
|
"Should reject USDC header");
|
||||||
|
|
||||||
|
// Too short data
|
||||||
|
uint8_t short_data[] = {0x28, 0xB5};
|
||||||
|
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(short_data, 2),
|
||||||
|
"Should reject too short data");
|
||||||
|
|
||||||
|
// Null data
|
||||||
|
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(nullptr, 0),
|
||||||
|
"Should reject null data");
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 2: Basic compression/decompression round-trip
|
||||||
|
int test_compression_roundtrip() {
|
||||||
|
std::cout << "\n=== Test: Compression Round-trip ===" << std::endl;
|
||||||
|
|
||||||
|
const char* test_data = "This is a test string for zstd compression. "
|
||||||
|
"It should be compressed and decompressed correctly.";
|
||||||
|
size_t test_size = strlen(test_data);
|
||||||
|
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
std::string err;
|
||||||
|
|
||||||
|
// Compress
|
||||||
|
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||||
|
reinterpret_cast<const uint8_t*>(test_data), test_size,
|
||||||
|
&compressed, 5, &err);
|
||||||
|
TEST_ASSERT(compress_ok, "Compression should succeed");
|
||||||
|
TEST_ASSERT(compressed.size() > 0, "Compressed size should be > 0");
|
||||||
|
TEST_ASSERT(compressed.size() < test_size, "Compressed data should be smaller");
|
||||||
|
|
||||||
|
// Verify magic number
|
||||||
|
TEST_ASSERT(tinyusdz::ZstdCompression::IsZstdCompressed(compressed.data(), compressed.size()),
|
||||||
|
"Compressed data should have zstd magic number");
|
||||||
|
|
||||||
|
// Get decompressed size
|
||||||
|
size_t decompressed_size = tinyusdz::ZstdCompression::GetDecompressedSize(
|
||||||
|
compressed.data(), compressed.size(), &err);
|
||||||
|
TEST_ASSERT(decompressed_size == test_size, "Decompressed size should match original");
|
||||||
|
|
||||||
|
// Decompress
|
||||||
|
std::vector<uint8_t> decompressed;
|
||||||
|
bool decompress_ok = tinyusdz::ZstdCompression::Decompress(
|
||||||
|
compressed.data(), compressed.size(), &decompressed, &err);
|
||||||
|
TEST_ASSERT(decompress_ok, "Decompression should succeed");
|
||||||
|
TEST_ASSERT(decompressed.size() == test_size, "Decompressed data size should match");
|
||||||
|
TEST_ASSERT(memcmp(decompressed.data(), test_data, test_size) == 0,
|
||||||
|
"Decompressed data should match original");
|
||||||
|
|
||||||
|
std::cout << "Compression ratio: " << test_size << " -> " << compressed.size()
|
||||||
|
<< " (" << (100.0 * compressed.size() / test_size) << "%)" << std::endl;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 3: Different compression levels
|
||||||
|
int test_compression_levels() {
|
||||||
|
std::cout << "\n=== Test: Compression Levels ===" << std::endl;
|
||||||
|
|
||||||
|
// Generate some compressible test data
|
||||||
|
std::string test_data;
|
||||||
|
for (int i = 0; i < 1000; i++) {
|
||||||
|
test_data += "Hello World! This is test data for compression level testing. ";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string err;
|
||||||
|
size_t prev_size = test_data.size() + 1; // Start larger than input
|
||||||
|
|
||||||
|
for (int level = 1; level <= 9; level += 4) {
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
bool ok = tinyusdz::ZstdCompression::Compress(
|
||||||
|
reinterpret_cast<const uint8_t*>(test_data.data()), test_data.size(),
|
||||||
|
&compressed, level, &err);
|
||||||
|
TEST_ASSERT(ok, "Compression at level " + std::to_string(level) + " should succeed");
|
||||||
|
|
||||||
|
std::cout << "Level " << level << ": " << test_data.size() << " -> "
|
||||||
|
<< compressed.size() << " bytes ("
|
||||||
|
<< (100.0 * compressed.size() / test_data.size()) << "%)" << std::endl;
|
||||||
|
|
||||||
|
// Higher levels should generally give same or better compression
|
||||||
|
// (not strictly enforced, but good to log)
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 4: Error handling for corrupt data
|
||||||
|
int test_corrupt_data_handling() {
|
||||||
|
std::cout << "\n=== Test: Corrupt Data Handling ===" << std::endl;
|
||||||
|
|
||||||
|
std::string err;
|
||||||
|
|
||||||
|
// Create corrupt data with valid magic but invalid content
|
||||||
|
uint8_t corrupt_data[] = {0x28, 0xB5, 0x2F, 0xFD, 0xFF, 0xFF, 0xFF, 0xFF};
|
||||||
|
std::vector<uint8_t> output;
|
||||||
|
|
||||||
|
bool ok = tinyusdz::ZstdCompression::Decompress(corrupt_data, 8, &output, &err);
|
||||||
|
TEST_ASSERT_FALSE(ok, "Decompression of corrupt data should fail");
|
||||||
|
TEST_ASSERT(err.size() > 0, "Error message should be provided");
|
||||||
|
|
||||||
|
std::cout << "Expected error: " << err << std::endl;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 5: GetCompressBound
|
||||||
|
int test_compress_bound() {
|
||||||
|
std::cout << "\n=== Test: GetCompressBound ===" << std::endl;
|
||||||
|
|
||||||
|
size_t bound = tinyusdz::ZstdCompression::GetCompressBound(1000);
|
||||||
|
TEST_ASSERT(bound > 1000, "Compress bound should be larger than input size");
|
||||||
|
|
||||||
|
bound = tinyusdz::ZstdCompression::GetCompressBound(0);
|
||||||
|
TEST_ASSERT(bound > 0, "Compress bound for 0 should still return some value");
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 6: USDA file round-trip with compression
|
||||||
|
int test_usda_roundtrip() {
|
||||||
|
std::cout << "\n=== Test: USDA Round-trip with Compression ===" << std::endl;
|
||||||
|
|
||||||
|
// Create a simple stage
|
||||||
|
tinyusdz::Stage stage;
|
||||||
|
stage.metas().doc = "Test USD file for zstd compression";
|
||||||
|
|
||||||
|
// Add a simple Xform prim
|
||||||
|
tinyusdz::Xform xform;
|
||||||
|
xform.name = "TestXform";
|
||||||
|
|
||||||
|
tinyusdz::Prim prim;
|
||||||
|
prim.set_data(xform);
|
||||||
|
stage.root_prims().push_back(prim);
|
||||||
|
|
||||||
|
// Export to string first
|
||||||
|
std::string usda_content = stage.ExportToString();
|
||||||
|
TEST_ASSERT(usda_content.size() > 0, "USDA export should produce content");
|
||||||
|
|
||||||
|
// Compress the USDA content
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
std::string err;
|
||||||
|
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||||
|
reinterpret_cast<const uint8_t*>(usda_content.data()), usda_content.size(),
|
||||||
|
&compressed, 5, &err);
|
||||||
|
TEST_ASSERT(compress_ok, "USDA compression should succeed");
|
||||||
|
|
||||||
|
// Decompress
|
||||||
|
std::vector<uint8_t> decompressed;
|
||||||
|
bool decompress_ok = tinyusdz::ZstdCompression::Decompress(
|
||||||
|
compressed.data(), compressed.size(), &decompressed, &err);
|
||||||
|
TEST_ASSERT(decompress_ok, "USDA decompression should succeed");
|
||||||
|
|
||||||
|
// Verify content matches
|
||||||
|
std::string decompressed_content(decompressed.begin(), decompressed.end());
|
||||||
|
TEST_ASSERT(decompressed_content == usda_content,
|
||||||
|
"Decompressed USDA content should match original");
|
||||||
|
|
||||||
|
std::cout << "USDA size: " << usda_content.size() << " -> " << compressed.size()
|
||||||
|
<< " bytes (" << (100.0 * compressed.size() / usda_content.size()) << "%)"
|
||||||
|
<< std::endl;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 7: IsZstdCompressed wrapper function in tinyusdz namespace
|
||||||
|
int test_tinyusdz_is_zstd_compressed() {
|
||||||
|
std::cout << "\n=== Test: tinyusdz::IsZstdCompressed ===" << std::endl;
|
||||||
|
|
||||||
|
// Valid zstd magic
|
||||||
|
uint8_t valid_magic[] = {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x00, 0x00};
|
||||||
|
TEST_ASSERT(tinyusdz::IsZstdCompressed(valid_magic, 8),
|
||||||
|
"tinyusdz::IsZstdCompressed should detect valid zstd magic");
|
||||||
|
|
||||||
|
// Invalid data
|
||||||
|
uint8_t invalid_data[] = {'#', 'u', 's', 'd', 'a'};
|
||||||
|
TEST_ASSERT_FALSE(tinyusdz::IsZstdCompressed(invalid_data, 5),
|
||||||
|
"tinyusdz::IsZstdCompressed should reject non-zstd data");
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 8: Load zstd-compressed USDA from memory
|
||||||
|
int test_load_compressed_usda_from_memory() {
|
||||||
|
std::cout << "\n=== Test: Load Compressed USDA from Memory ===" << std::endl;
|
||||||
|
|
||||||
|
// Create a minimal USDA content
|
||||||
|
const char* usda_content = R"(#usda 1.0
|
||||||
|
(
|
||||||
|
doc = "Test compressed USDA"
|
||||||
|
)
|
||||||
|
|
||||||
|
def Xform "TestXform"
|
||||||
|
{
|
||||||
|
}
|
||||||
|
)";
|
||||||
|
size_t usda_size = strlen(usda_content);
|
||||||
|
|
||||||
|
// Compress it
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
std::string err;
|
||||||
|
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||||
|
reinterpret_cast<const uint8_t*>(usda_content), usda_size,
|
||||||
|
&compressed, 5, &err);
|
||||||
|
TEST_ASSERT(compress_ok, "Compression should succeed");
|
||||||
|
|
||||||
|
// Load as USD from memory
|
||||||
|
tinyusdz::Stage stage;
|
||||||
|
std::string warn;
|
||||||
|
bool load_ok = tinyusdz::LoadUSDFromMemory(
|
||||||
|
compressed.data(), compressed.size(), "", &stage, &warn, &err);
|
||||||
|
TEST_ASSERT(load_ok, "Loading compressed USDA from memory should succeed");
|
||||||
|
|
||||||
|
// Verify the loaded content
|
||||||
|
TEST_ASSERT(stage.metas().doc.value == "Test compressed USDA",
|
||||||
|
"Document string should match");
|
||||||
|
TEST_ASSERT(stage.root_prims().size() > 0, "Should have at least one root prim");
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 9: Memory budget enforcement
|
||||||
|
int test_memory_budget() {
|
||||||
|
std::cout << "\n=== Test: Memory Budget Enforcement ===" << std::endl;
|
||||||
|
|
||||||
|
// Create a large piece of test data
|
||||||
|
std::string large_data(10 * 1024 * 1024, 'X'); // 10MB of data
|
||||||
|
|
||||||
|
// Compress it
|
||||||
|
std::vector<uint8_t> compressed;
|
||||||
|
std::string err;
|
||||||
|
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||||
|
reinterpret_cast<const uint8_t*>(large_data.data()), large_data.size(),
|
||||||
|
&compressed, 1, &err); // Use level 1 for speed
|
||||||
|
TEST_ASSERT(compress_ok, "Compression should succeed");
|
||||||
|
|
||||||
|
// Try to load with a small memory budget
|
||||||
|
tinyusdz::USDLoadOptions options;
|
||||||
|
options.max_memory_limit_in_mb = 1; // Only 1MB allowed
|
||||||
|
|
||||||
|
tinyusdz::Stage stage;
|
||||||
|
std::string warn;
|
||||||
|
err.clear();
|
||||||
|
|
||||||
|
// This should fail because decompressed size exceeds the memory limit
|
||||||
|
// Note: This test may not work perfectly because the compressed data
|
||||||
|
// isn't valid USD, but it tests the memory check path
|
||||||
|
bool load_ok = tinyusdz::LoadUSDFromMemory(
|
||||||
|
compressed.data(), compressed.size(), "", &stage, &warn, &err, options);
|
||||||
|
|
||||||
|
// Should fail due to memory limit
|
||||||
|
TEST_ASSERT_FALSE(load_ok, "Loading should fail due to memory limit");
|
||||||
|
std::cout << "Expected error: " << err << std::endl;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 10: Write compressed USDA file
|
||||||
|
int test_write_compressed_usda() {
|
||||||
|
std::cout << "\n=== Test: Write Compressed USDA File ===" << std::endl;
|
||||||
|
|
||||||
|
// Create a simple stage
|
||||||
|
tinyusdz::Stage stage;
|
||||||
|
stage.metas().doc = "Test compressed USDA write";
|
||||||
|
|
||||||
|
tinyusdz::Xform xform;
|
||||||
|
xform.name = "WriteTest";
|
||||||
|
|
||||||
|
tinyusdz::Prim prim;
|
||||||
|
prim.set_data(xform);
|
||||||
|
stage.root_prims().push_back(prim);
|
||||||
|
|
||||||
|
// Write with compression (using options)
|
||||||
|
tinyusdz::USDWriteOptions options;
|
||||||
|
options.use_zstd_compression = true;
|
||||||
|
options.zstd_compression_level = 5;
|
||||||
|
|
||||||
|
std::string warn, err;
|
||||||
|
bool write_ok = tinyusdz::usda::SaveAsUSDA(
|
||||||
|
"test_output.usda.zst", stage, &warn, &err, options);
|
||||||
|
TEST_ASSERT(write_ok, "Writing compressed USDA should succeed");
|
||||||
|
|
||||||
|
// Read back the file
|
||||||
|
std::ifstream ifs("test_output.usda.zst", std::ios::binary);
|
||||||
|
TEST_ASSERT(ifs.is_open(), "Output file should exist");
|
||||||
|
|
||||||
|
std::vector<uint8_t> file_content((std::istreambuf_iterator<char>(ifs)),
|
||||||
|
std::istreambuf_iterator<char>());
|
||||||
|
ifs.close();
|
||||||
|
|
||||||
|
// Verify it's zstd compressed
|
||||||
|
TEST_ASSERT(tinyusdz::IsZstdCompressed(file_content.data(), file_content.size()),
|
||||||
|
"Output file should be zstd compressed");
|
||||||
|
|
||||||
|
// Load it back
|
||||||
|
tinyusdz::Stage loaded_stage;
|
||||||
|
bool load_ok = tinyusdz::LoadUSDFromMemory(
|
||||||
|
file_content.data(), file_content.size(), "", &loaded_stage, &warn, &err);
|
||||||
|
TEST_ASSERT(load_ok, "Loading written compressed file should succeed");
|
||||||
|
|
||||||
|
// Verify content
|
||||||
|
TEST_ASSERT(loaded_stage.metas().doc.value == "Test compressed USDA write",
|
||||||
|
"Loaded doc should match");
|
||||||
|
|
||||||
|
// Clean up test file
|
||||||
|
std::remove("test_output.usda.zst");
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char** argv) {
|
||||||
|
(void)argc;
|
||||||
|
(void)argv;
|
||||||
|
|
||||||
|
std::cout << "===== Zstd USD Compression Feature Tests =====" << std::endl;
|
||||||
|
|
||||||
|
int failures = 0;
|
||||||
|
|
||||||
|
failures += test_magic_number_detection();
|
||||||
|
failures += test_compression_roundtrip();
|
||||||
|
failures += test_compression_levels();
|
||||||
|
failures += test_corrupt_data_handling();
|
||||||
|
failures += test_compress_bound();
|
||||||
|
failures += test_usda_roundtrip();
|
||||||
|
failures += test_tinyusdz_is_zstd_compressed();
|
||||||
|
failures += test_load_compressed_usda_from_memory();
|
||||||
|
failures += test_memory_budget();
|
||||||
|
failures += test_write_compressed_usda();
|
||||||
|
|
||||||
|
std::cout << "\n===== Summary =====" << std::endl;
|
||||||
|
if (failures == 0) {
|
||||||
|
std::cout << "All tests PASSED!" << std::endl;
|
||||||
|
return 0;
|
||||||
|
} else {
|
||||||
|
std::cout << failures << " test(s) FAILED!" << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user