mirror of
https://github.com/lighttransport/tinyusdz.git
synced 2026-01-18 01:11:17 +01:00
Add zstd compression support for USD files
Add file-level zstd compression/decompression for USD files (USDA, USDC, USDZ). This allows transparent loading of zstd-compressed USD files and optional compression when writing. Features: - Automatic detection by zstd magic number (0x28 0xB5 0x2F 0xFD) - Transparent decompression in LoadUSDFromMemory() - Optional compression in SaveAsUSDA() and SaveAsUSDCToFile() - Auto-detection of .zst file extension for compression - Configurable compression level (1-22, default 5) - Memory budget enforcement during decompression - Uses amalgamated single-file zstd library (v1.5.6) Build options: - TINYUSDZ_WITH_ZSTD_COMPRESSION=ON (default) - TINYUSDZ_USE_SYSTEM_ZSTD=ON (use system library) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -186,6 +186,11 @@ endif ()
|
||||
option(TINYUSDZ_USE_SYSTEM_ZLIB
|
||||
"Use system's zlib instead of miniz for TinyEXR/TIFF" OFF)
|
||||
|
||||
# -- Zstd compression --
|
||||
option(TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
"Enable zstd compression for USD files" ON)
|
||||
option(TINYUSDZ_USE_SYSTEM_ZSTD
|
||||
"Use system's zstd instead of bundled version" OFF)
|
||||
|
||||
option(
|
||||
TINYUSDZ_PRODUCTION_BUILD
|
||||
@@ -667,6 +672,21 @@ if(TINYUSDZ_WITH_TIFF OR TINYUSDZ_WITH_EXR)
|
||||
|
||||
endif()
|
||||
|
||||
# -- Zstd compression --
|
||||
if(TINYUSDZ_WITH_ZSTD_COMPRESSION)
|
||||
if(NOT TINYUSDZ_USE_SYSTEM_ZSTD)
|
||||
list(APPEND TINYUSDZ_DEP_SOURCES ${PROJECT_SOURCE_DIR}/src/external/zstd.c)
|
||||
# Disable ASM to avoid linker issues, suppress warnings for third-party code
|
||||
set_source_files_properties(
|
||||
${PROJECT_SOURCE_DIR}/src/external/zstd.c
|
||||
PROPERTIES COMPILE_DEFINITIONS "ZSTD_DISABLE_ASM=1"
|
||||
COMPILE_FLAGS "-w")
|
||||
else()
|
||||
find_package(zstd REQUIRED)
|
||||
endif()
|
||||
list(APPEND TINYUSDZ_DEP_SOURCES ${PROJECT_SOURCE_DIR}/src/zstd-compression.cc)
|
||||
endif()
|
||||
|
||||
if(TINYUSDZ_WITH_ALAC_AUDIO)
|
||||
list(APPEND TINYUSDZ_DEP_SOURCES
|
||||
${PROJECT_SOURCE_DIR}/src/external/alac/codec/EndianPortable.c
|
||||
@@ -1341,6 +1361,13 @@ foreach(TINYUSDZ_LIB_TARGET ${TINYUSDZ_LIBS})
|
||||
target_compile_definitions(${TINYUSDZ_LIB_TARGET} PRIVATE "TINYUSDZ_NO_STB_IMAGE_WRITE_IMPLEMENTATION")
|
||||
endif()
|
||||
|
||||
if(TINYUSDZ_WITH_ZSTD_COMPRESSION)
|
||||
target_compile_definitions(${TINYUSDZ_LIB_TARGET} PRIVATE "TINYUSDZ_WITH_ZSTD_COMPRESSION")
|
||||
if(TINYUSDZ_USE_SYSTEM_ZSTD)
|
||||
target_link_libraries(${TINYUSDZ_LIB_TARGET} PUBLIC zstd::zstd)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
target_include_directories(${TINYUSDZ_LIB_TARGET}
|
||||
PRIVATE ${PROJECT_SOURCE_DIR}/src)
|
||||
|
||||
|
||||
BIN
models/suzanne-pbr.usda.zst
Normal file
BIN
models/suzanne-pbr.usda.zst
Normal file
Binary file not shown.
51006
src/external/zstd.c
vendored
Normal file
51006
src/external/zstd.c
vendored
Normal file
File diff suppressed because it is too large
Load Diff
3089
src/external/zstd.h
vendored
Normal file
3089
src/external/zstd.h
vendored
Normal file
File diff suppressed because it is too large
Load Diff
@@ -26,6 +26,7 @@
|
||||
#include "integerCoding.h"
|
||||
#include "io-util.hh"
|
||||
#include "lz4-compression.hh"
|
||||
#include "zstd-compression.hh"
|
||||
#include "pprinter.hh"
|
||||
#include "str-util.hh"
|
||||
#include "stream-reader.hh"
|
||||
@@ -920,6 +921,51 @@ bool LoadUSDFromMemory(const uint8_t *addr, const size_t length,
|
||||
const std::string &base_dir, Stage *stage,
|
||||
std::string *warn, std::string *err,
|
||||
const USDLoadOptions &options) {
|
||||
// Check for zstd-compressed data first (file-level compression)
|
||||
if (IsZstdCompressed(addr, length)) {
|
||||
DCOUT("Detected as zstd-compressed USD.");
|
||||
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
// Get decompressed size for memory budget check
|
||||
std::string zstd_err;
|
||||
size_t decompressed_size = ZstdCompression::GetDecompressedSize(addr, length, &zstd_err);
|
||||
if (decompressed_size == 0) {
|
||||
if (err) {
|
||||
(*err) += "Failed to get zstd decompressed size: " + zstd_err + "\n";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check against memory budget
|
||||
size_t max_length = size_t(1024) * size_t(1024) * size_t(options.max_memory_limit_in_mb);
|
||||
if (decompressed_size > max_length) {
|
||||
if (err) {
|
||||
(*err) += "Decompressed USD size (" + std::to_string(decompressed_size) +
|
||||
" bytes) exceeds memory limit (" + std::to_string(max_length) + " bytes)\n";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Decompress
|
||||
std::vector<uint8_t> decompressed_data;
|
||||
if (!ZstdCompression::Decompress(addr, length, &decompressed_data, &zstd_err)) {
|
||||
if (err) {
|
||||
(*err) += "Failed to decompress zstd data: " + zstd_err + "\n";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Recursively call LoadUSDFromMemory with decompressed data
|
||||
return LoadUSDFromMemory(decompressed_data.data(), decompressed_data.size(),
|
||||
base_dir, stage, warn, err, options);
|
||||
#else
|
||||
if (err) {
|
||||
(*err) += "zstd-compressed USD file detected, but zstd compression support is not enabled. "
|
||||
"Rebuild with TINYUSDZ_WITH_ZSTD_COMPRESSION=ON.\n";
|
||||
}
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (IsUSDC(addr, length)) {
|
||||
DCOUT("Detected as USDC.");
|
||||
return LoadUSDCFromMemory(addr, length, base_dir, stage, warn, err,
|
||||
@@ -1086,6 +1132,10 @@ bool IsUSDZ(const uint8_t *addr, const size_t length) {
|
||||
return ParseUSDZHeader(addr, length, /* [out] assets */ nullptr, &warn, &err);
|
||||
}
|
||||
|
||||
bool IsZstdCompressed(const uint8_t *addr, const size_t length) {
|
||||
return ZstdCompression::IsZstdCompressed(addr, length);
|
||||
}
|
||||
|
||||
bool IsUSD(const std::string &filename, std::string *detected_format) {
|
||||
if (IsUSDA(filename)) {
|
||||
if (detected_format) {
|
||||
|
||||
@@ -138,13 +138,24 @@ struct USDLoadOptions {
|
||||
// - Realtime(moderate resource size limit)
|
||||
// - DCC(for data conversion. Unlimited resource size)
|
||||
|
||||
#if 0 // TODO
|
||||
//struct USDWriteOptions
|
||||
//{
|
||||
//
|
||||
//
|
||||
//};
|
||||
#endif
|
||||
///
|
||||
/// Options for writing USD files.
|
||||
///
|
||||
struct USDWriteOptions {
|
||||
///
|
||||
/// Enable zstd compression for output file.
|
||||
/// When enabled, the entire USD file is wrapped with zstd compression.
|
||||
/// Also auto-detected when filename ends with ".zst" extension.
|
||||
///
|
||||
bool use_zstd_compression{false};
|
||||
|
||||
///
|
||||
/// Zstd compression level (1-22).
|
||||
/// Higher values give better compression but slower speed.
|
||||
/// Default is 5 (good balance of speed and ratio).
|
||||
///
|
||||
int zstd_compression_level{5};
|
||||
};
|
||||
|
||||
//
|
||||
|
||||
@@ -469,6 +480,10 @@ bool IsUSDC(const uint8_t *addr, const size_t length);
|
||||
bool IsUSDZ(const std::string &filename);
|
||||
bool IsUSDZ(const uint8_t *addr, const size_t length);
|
||||
|
||||
// Test if input is zstd-compressed data (by magic number).
|
||||
// This is for file-level compression, not internal USDC LZ4 compression.
|
||||
bool IsZstdCompressed(const uint8_t *addr, const size_t length);
|
||||
|
||||
} // namespace tinyusdz
|
||||
|
||||
#endif // TINYUSDZ_HH_
|
||||
|
||||
@@ -17,46 +17,131 @@
|
||||
#include "value-pprint.hh"
|
||||
#include "tinyusdz.hh"
|
||||
#include "io-util.hh"
|
||||
#include "str-util.hh"
|
||||
#include "zstd-compression.hh"
|
||||
|
||||
namespace tinyusdz {
|
||||
namespace usda {
|
||||
|
||||
namespace {
|
||||
|
||||
// Check if filename ends with ".zst" extension (case-insensitive)
|
||||
bool HasZstdExtension(const std::string &filename) {
|
||||
if (filename.size() < 4) return false;
|
||||
std::string ext = filename.substr(filename.size() - 4);
|
||||
return (ext == ".zst" || ext == ".ZST");
|
||||
}
|
||||
|
||||
#if defined(_WIN32)
|
||||
bool HasZstdExtension(const std::wstring &filename) {
|
||||
if (filename.size() < 4) return false;
|
||||
std::wstring ext = filename.substr(filename.size() - 4);
|
||||
return (ext == L".zst" || ext == L".ZST");
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace
|
||||
|
||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage,
|
||||
std::string *warn, std::string *err) {
|
||||
|
||||
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err) {
|
||||
(void)warn;
|
||||
(void)err;
|
||||
|
||||
// TODO: Handle warn and err on export.
|
||||
std::string s = stage.ExportToString();
|
||||
|
||||
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
||||
if (!output) {
|
||||
if (err) {
|
||||
(*err) = "output parameter is null.\n";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::cout << "Wrote to [" << filename << "]\n";
|
||||
*output = stage.ExportToString();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage,
|
||||
std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options) {
|
||||
|
||||
(void)warn;
|
||||
|
||||
// Export stage to string
|
||||
std::string s = stage.ExportToString();
|
||||
|
||||
// Check if we should use zstd compression
|
||||
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
|
||||
|
||||
if (use_compression) {
|
||||
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
// Compress the data
|
||||
std::vector<uint8_t> compressed;
|
||||
if (!ZstdCompression::Compress(reinterpret_cast<const uint8_t*>(s.data()), s.size(),
|
||||
&compressed, options.zstd_compression_level, err)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!io::WriteWholeFile(filename, compressed.data(), compressed.size(), err)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::cout << "Wrote zstd-compressed USDA to [" << filename << "] ("
|
||||
<< s.size() << " -> " << compressed.size() << " bytes)\n";
|
||||
#else
|
||||
if (err) {
|
||||
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
|
||||
}
|
||||
return false;
|
||||
#endif
|
||||
} else {
|
||||
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::cout << "Wrote USDA to [" << filename << "]\n";
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#if defined(_WIN32)
|
||||
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage,
|
||||
std::string *warn, std::string *err) {
|
||||
std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options) {
|
||||
|
||||
(void)warn;
|
||||
|
||||
// TODO: Handle warn and err on export.
|
||||
// Export stage to string
|
||||
std::string s = stage.ExportToString();
|
||||
|
||||
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
||||
return false;
|
||||
}
|
||||
// Check if we should use zstd compression
|
||||
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
|
||||
|
||||
std::wcout << "Wrote to [" << filename << "]\n";
|
||||
if (use_compression) {
|
||||
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
// Compress the data
|
||||
std::vector<uint8_t> compressed;
|
||||
if (!ZstdCompression::Compress(reinterpret_cast<const uint8_t*>(s.data()), s.size(),
|
||||
&compressed, options.zstd_compression_level, err)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!io::WriteWholeFile(filename, compressed.data(), compressed.size(), err)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::wcout << L"Wrote zstd-compressed USDA to [" << filename << L"] ("
|
||||
<< s.size() << L" -> " << compressed.size() << L" bytes)\n";
|
||||
#else
|
||||
if (err) {
|
||||
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
|
||||
}
|
||||
return false;
|
||||
#endif
|
||||
} else {
|
||||
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::wcout << L"Wrote USDA to [" << filename << L"]\n";
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -71,9 +156,9 @@ bool SaveAsUSDA(const std::wstring &filename, const Stage &stage,
|
||||
namespace tinyusdz {
|
||||
namespace usda {
|
||||
|
||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err) {
|
||||
(void)filename;
|
||||
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err) {
|
||||
(void)stage;
|
||||
(void)output;
|
||||
(void)warn;
|
||||
|
||||
if (err) {
|
||||
@@ -82,7 +167,33 @@ bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *wa
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options) {
|
||||
(void)filename;
|
||||
(void)stage;
|
||||
(void)warn;
|
||||
(void)options;
|
||||
|
||||
if (err) {
|
||||
(*err) = "USDA Writer feature is disabled in this build.\n";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
#if defined(_WIN32)
|
||||
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options) {
|
||||
(void)filename;
|
||||
(void)stage;
|
||||
(void)warn;
|
||||
(void)options;
|
||||
|
||||
if (err) {
|
||||
(*err) = "USDA Writer feature is disabled in this build.\n";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace usda
|
||||
} // namespace tinyusdz
|
||||
|
||||
@@ -16,18 +16,34 @@ namespace usda {
|
||||
/// Save scene as USDA(ASCII)
|
||||
///
|
||||
/// @param[in] filename USDA filename(UTF-8). WideChar(Unicode) represented as std::string is supported on Windows.
|
||||
/// If filename ends with ".zst", zstd compression is automatically enabled.
|
||||
/// @param[in] stage Stage(scene graph).
|
||||
/// @param[out] warn Warning message
|
||||
/// @param[out] err Error message
|
||||
/// @param[in] options Write options (optional). Includes zstd compression settings.
|
||||
///
|
||||
/// @return true upon success.
|
||||
///
|
||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options = USDWriteOptions());
|
||||
|
||||
#if defined(_WIN32)
|
||||
// WideChar(UNICODE) filename version.
|
||||
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options = USDWriteOptions());
|
||||
#endif
|
||||
|
||||
///
|
||||
/// Export stage as USDA string.
|
||||
///
|
||||
/// @param[in] stage Stage(scene graph).
|
||||
/// @param[out] output USDA string output
|
||||
/// @param[out] warn Warning message
|
||||
/// @param[out] err Error message
|
||||
///
|
||||
/// @return true upon success.
|
||||
///
|
||||
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err);
|
||||
|
||||
#if defined(_WIN32)
|
||||
// WideChar(UNICODE) filename version.
|
||||
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err);
|
||||
#endif
|
||||
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err);
|
||||
|
||||
} // namespace usda
|
||||
} // namespace tinyusdz
|
||||
|
||||
@@ -41,6 +41,7 @@
|
||||
#include "crate-format.hh"
|
||||
#include "io-util.hh"
|
||||
#include "lz4-compression.hh"
|
||||
#include "zstd-compression.hh"
|
||||
#include "token-type.hh"
|
||||
|
||||
#include "common-macros.inc"
|
||||
@@ -50,6 +51,13 @@ namespace usdc {
|
||||
|
||||
namespace {
|
||||
|
||||
// Check if filename ends with ".zst" extension (case-insensitive)
|
||||
bool HasZstdExtension(const std::string &filename) {
|
||||
if (filename.size() < 4) return false;
|
||||
std::string ext = filename.substr(filename.size() - 4);
|
||||
return (ext == ".zst" || ext == ".ZST");
|
||||
}
|
||||
|
||||
constexpr size_t kSectionNameMaxLength = 15;
|
||||
|
||||
#ifdef _WIN32
|
||||
@@ -480,11 +488,13 @@ class Writer {
|
||||
} // namespace
|
||||
|
||||
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||
std::string *warn, std::string *err) {
|
||||
std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options) {
|
||||
#ifdef __ANDROID__
|
||||
(void)filename;
|
||||
(void)stage;
|
||||
(void)warn;
|
||||
(void)options;
|
||||
|
||||
if (err) {
|
||||
(*err) += "Saving USDC to a file is not supported for Android platform(at the moment).\n";
|
||||
@@ -498,6 +508,30 @@ bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if we should use zstd compression
|
||||
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
|
||||
|
||||
const uint8_t *write_data = output.data();
|
||||
size_t write_size = output.size();
|
||||
std::vector<uint8_t> compressed;
|
||||
|
||||
if (use_compression) {
|
||||
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
if (!ZstdCompression::Compress(output.data(), output.size(),
|
||||
&compressed, options.zstd_compression_level, err)) {
|
||||
return false;
|
||||
}
|
||||
write_data = compressed.data();
|
||||
write_size = compressed.size();
|
||||
std::cout << "Compressing USDC with zstd (" << output.size() << " -> " << compressed.size() << " bytes)\n";
|
||||
#else
|
||||
if (err) {
|
||||
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
|
||||
}
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
#if defined(_MSC_VER) || defined(__GLIBCXX__) || defined(__clang__)
|
||||
FILE *fp = nullptr;
|
||||
@@ -530,9 +564,11 @@ bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||
}
|
||||
#endif
|
||||
|
||||
size_t n = fwrite(output.data(), /* size */ 1, /* count */ output.size(), fp);
|
||||
if (n < output.size()) {
|
||||
// TODO: Retry writing data when n < output.size()
|
||||
size_t n = fwrite(write_data, /* size */ 1, /* count */ write_size, fp);
|
||||
fclose(fp);
|
||||
|
||||
if (n < write_size) {
|
||||
// TODO: Retry writing data when n < write_size
|
||||
|
||||
if (err) {
|
||||
(*err) += "Failed to write data to a file.\n";
|
||||
@@ -568,10 +604,12 @@ namespace tinyusdz {
|
||||
namespace usdc {
|
||||
|
||||
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||
std::string *warn, std::string *err) {
|
||||
std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options) {
|
||||
(void)filename;
|
||||
(void)stage;
|
||||
(void)warn;
|
||||
(void)options;
|
||||
|
||||
if (err) {
|
||||
(*err) = "USDC writer feature is disabled in this build.\n";
|
||||
|
||||
@@ -19,15 +19,17 @@ namespace usdc {
|
||||
///
|
||||
/// Save scene as USDC(binary) to a file
|
||||
///
|
||||
/// @param[in] filename USDC filename
|
||||
/// @param[in] filename USDC filename. If filename ends with ".zst", zstd compression is auto-enabled.
|
||||
/// @param[in] stage Stage
|
||||
/// @param[out] warn Warning message
|
||||
/// @param[out] err Error message
|
||||
/// @param[in] options Write options (optional). Includes zstd compression settings.
|
||||
///
|
||||
/// @return true upon success.
|
||||
///
|
||||
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
|
||||
std::string *warn, std::string *err);
|
||||
std::string *warn, std::string *err,
|
||||
const USDWriteOptions &options = USDWriteOptions());
|
||||
|
||||
///
|
||||
/// Save scene as USDC(binary) to a memory
|
||||
|
||||
175
src/zstd-compression.cc
Normal file
175
src/zstd-compression.cc
Normal file
@@ -0,0 +1,175 @@
|
||||
// SPDX-License-Identifier: Apache 2.0
|
||||
// Copyright 2024-Present Light Transport Entertainment Inc.
|
||||
//
|
||||
// Zstd compression wrapper implementation for TinyUSDZ
|
||||
|
||||
#include "zstd-compression.hh"
|
||||
|
||||
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
|
||||
// Use amalgamated single-file zstd
|
||||
#include "external/zstd.h"
|
||||
|
||||
namespace tinyusdz {
|
||||
|
||||
bool ZstdCompression::IsZstdCompressed(const uint8_t *data, size_t length) {
|
||||
if (!data || length < 4) {
|
||||
return false;
|
||||
}
|
||||
return (data[0] == kZstdMagic[0] && data[1] == kZstdMagic[1] &&
|
||||
data[2] == kZstdMagic[2] && data[3] == kZstdMagic[3]);
|
||||
}
|
||||
|
||||
size_t ZstdCompression::GetDecompressedSize(const uint8_t *compressed,
|
||||
size_t compressedSize,
|
||||
std::string *err) {
|
||||
if (!compressed || compressedSize == 0) {
|
||||
if (err) *err = "Invalid compressed data";
|
||||
return 0;
|
||||
}
|
||||
|
||||
unsigned long long frameContentSize =
|
||||
ZSTD_getFrameContentSize(compressed, compressedSize);
|
||||
|
||||
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR) {
|
||||
if (err) *err = "Not a valid zstd compressed frame";
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
|
||||
if (err) *err = "Zstd frame does not contain content size";
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Check for size overflow
|
||||
if (frameContentSize > std::numeric_limits<size_t>::max()) {
|
||||
if (err) *err = "Decompressed size exceeds size_t range";
|
||||
return 0;
|
||||
}
|
||||
|
||||
return static_cast<size_t>(frameContentSize);
|
||||
}
|
||||
|
||||
bool ZstdCompression::Decompress(const uint8_t *compressed,
|
||||
size_t compressedSize,
|
||||
std::vector<uint8_t> *output,
|
||||
std::string *err) {
|
||||
if (!compressed || compressedSize == 0) {
|
||||
if (err) *err = "Invalid compressed data";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!output) {
|
||||
if (err) *err = "Output buffer is null";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get decompressed size first
|
||||
size_t decompressedSize = GetDecompressedSize(compressed, compressedSize, err);
|
||||
if (decompressedSize == 0) {
|
||||
// err already set by GetDecompressedSize
|
||||
return false;
|
||||
}
|
||||
|
||||
// Allocate output buffer
|
||||
// Note: TinyUSDZ has exceptions disabled, so we check capacity instead
|
||||
output->clear();
|
||||
if (decompressedSize > output->max_size()) {
|
||||
if (err) *err = "Decompressed size exceeds maximum allocatable size";
|
||||
return false;
|
||||
}
|
||||
output->resize(decompressedSize);
|
||||
if (output->size() != decompressedSize) {
|
||||
if (err) *err = "Failed to allocate memory for decompression";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Decompress
|
||||
size_t result =
|
||||
ZSTD_decompress(output->data(), decompressedSize, compressed, compressedSize);
|
||||
|
||||
if (ZSTD_isError(result)) {
|
||||
if (err) {
|
||||
*err = "Zstd decompression failed: ";
|
||||
*err += ZSTD_getErrorName(result);
|
||||
}
|
||||
output->clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
// Verify decompressed size matches expected
|
||||
if (result != decompressedSize) {
|
||||
if (err) {
|
||||
*err = "Decompressed size mismatch: expected " +
|
||||
std::to_string(decompressedSize) + ", got " + std::to_string(result);
|
||||
}
|
||||
output->clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ZstdCompression::Compress(const uint8_t *input, size_t inputSize,
|
||||
std::vector<uint8_t> *output,
|
||||
int compressionLevel, std::string *err) {
|
||||
if (!input || inputSize == 0) {
|
||||
if (err) *err = "Invalid input data";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!output) {
|
||||
if (err) *err = "Output buffer is null";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Clamp compression level to valid range
|
||||
if (compressionLevel < 1) compressionLevel = 1;
|
||||
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
|
||||
|
||||
// Get maximum compressed size
|
||||
size_t maxCompressedSize = ZSTD_compressBound(inputSize);
|
||||
|
||||
// Allocate output buffer
|
||||
// Note: TinyUSDZ has exceptions disabled, so we check capacity instead
|
||||
output->clear();
|
||||
if (maxCompressedSize > output->max_size()) {
|
||||
if (err) *err = "Compressed buffer size exceeds maximum allocatable size";
|
||||
return false;
|
||||
}
|
||||
output->resize(maxCompressedSize);
|
||||
if (output->size() != maxCompressedSize) {
|
||||
if (err) *err = "Failed to allocate memory for compression";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compress
|
||||
size_t result =
|
||||
ZSTD_compress(output->data(), maxCompressedSize, input, inputSize,
|
||||
compressionLevel);
|
||||
|
||||
if (ZSTD_isError(result)) {
|
||||
if (err) {
|
||||
*err = "Zstd compression failed: ";
|
||||
*err += ZSTD_getErrorName(result);
|
||||
}
|
||||
output->clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
// Shrink to actual compressed size
|
||||
output->resize(result);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t ZstdCompression::GetCompressBound(size_t inputSize) {
|
||||
return ZSTD_compressBound(inputSize);
|
||||
}
|
||||
|
||||
} // namespace tinyusdz
|
||||
|
||||
#endif // TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
116
src/zstd-compression.hh
Normal file
116
src/zstd-compression.hh
Normal file
@@ -0,0 +1,116 @@
|
||||
// SPDX-License-Identifier: Apache 2.0
|
||||
// Copyright 2024-Present Light Transport Entertainment Inc.
|
||||
//
|
||||
// Zstd compression wrapper for TinyUSDZ
|
||||
// Provides file-level zstd compression/decompression for USD files
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifndef ZSTD_COMPRESSION_HH_
|
||||
#define ZSTD_COMPRESSION_HH_
|
||||
|
||||
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace tinyusdz {
|
||||
|
||||
///
|
||||
/// Zstd compression utility class.
|
||||
/// Provides static methods for compressing and decompressing data using zstd.
|
||||
///
|
||||
class ZstdCompression {
|
||||
public:
|
||||
///
|
||||
/// Check if data starts with zstd magic number (0x28 0xB5 0x2F 0xFD)
|
||||
/// @param[in] data Pointer to the data to check
|
||||
/// @param[in] length Length of the data in bytes
|
||||
/// @return true if data starts with zstd magic number
|
||||
///
|
||||
static bool IsZstdCompressed(const uint8_t *data, size_t length);
|
||||
|
||||
///
|
||||
/// Get the decompressed size of zstd-compressed data.
|
||||
/// This can be used to check memory budget before decompression.
|
||||
/// @param[in] compressed Pointer to compressed data
|
||||
/// @param[in] compressedSize Size of compressed data in bytes
|
||||
/// @param[out] err Error message if size cannot be determined
|
||||
/// @return Decompressed size in bytes, or 0 on error
|
||||
///
|
||||
static size_t GetDecompressedSize(const uint8_t *compressed,
|
||||
size_t compressedSize, std::string *err);
|
||||
|
||||
///
|
||||
/// Decompress zstd-compressed data.
|
||||
/// @param[in] compressed Pointer to compressed data
|
||||
/// @param[in] compressedSize Size of compressed data in bytes
|
||||
/// @param[out] output Vector to store decompressed data
|
||||
/// @param[out] err Error message on failure
|
||||
/// @return true on success, false on failure
|
||||
///
|
||||
static bool Decompress(const uint8_t *compressed, size_t compressedSize,
|
||||
std::vector<uint8_t> *output, std::string *err);
|
||||
|
||||
///
|
||||
/// Compress data using zstd.
|
||||
/// @param[in] input Pointer to input data
|
||||
/// @param[in] inputSize Size of input data in bytes
|
||||
/// @param[out] output Vector to store compressed data
|
||||
/// @param[in] compressionLevel Compression level (1-22, default 5)
|
||||
/// @param[out] err Error message on failure
|
||||
/// @return true on success, false on failure
|
||||
///
|
||||
static bool Compress(const uint8_t *input, size_t inputSize,
|
||||
std::vector<uint8_t> *output, int compressionLevel,
|
||||
std::string *err);
|
||||
|
||||
///
|
||||
/// Get the maximum compressed size for a given input size.
|
||||
/// @param[in] inputSize Size of input data in bytes
|
||||
/// @return Maximum possible compressed size
|
||||
///
|
||||
static size_t GetCompressBound(size_t inputSize);
|
||||
|
||||
/// Default compression level (5 is a good balance of speed and ratio)
|
||||
static constexpr int kDefaultCompressionLevel = 5;
|
||||
|
||||
/// Zstd magic number bytes
|
||||
static constexpr uint8_t kZstdMagic[4] = {0x28, 0xB5, 0x2F, 0xFD};
|
||||
};
|
||||
|
||||
} // namespace tinyusdz
|
||||
|
||||
#else // !TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
|
||||
// Stub implementation when zstd compression is disabled
|
||||
namespace tinyusdz {
|
||||
|
||||
class ZstdCompression {
|
||||
public:
|
||||
static bool IsZstdCompressed(const uint8_t *, size_t) { return false; }
|
||||
static size_t GetDecompressedSize(const uint8_t *, size_t, std::string *err) {
|
||||
if (err) *err = "Zstd compression support not enabled";
|
||||
return 0;
|
||||
}
|
||||
static bool Decompress(const uint8_t *, size_t, std::vector<uint8_t> *,
|
||||
std::string *err) {
|
||||
if (err) *err = "Zstd compression support not enabled";
|
||||
return false;
|
||||
}
|
||||
static bool Compress(const uint8_t *, size_t, std::vector<uint8_t> *, int,
|
||||
std::string *err) {
|
||||
if (err) *err = "Zstd compression support not enabled";
|
||||
return false;
|
||||
}
|
||||
static size_t GetCompressBound(size_t) { return 0; }
|
||||
static constexpr int kDefaultCompressionLevel = 5;
|
||||
};
|
||||
|
||||
} // namespace tinyusdz
|
||||
|
||||
#endif // TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
|
||||
#endif // ZSTD_COMPRESSION_HH_
|
||||
76
tests/feat/zstdusd/Makefile
Normal file
76
tests/feat/zstdusd/Makefile
Normal file
@@ -0,0 +1,76 @@
|
||||
# Makefile for zstd USD compression feature tests
|
||||
# This is a standalone Makefile that builds and runs the zstd compression tests
|
||||
|
||||
CXX = g++
|
||||
CXXFLAGS = -std=c++14 -O2 -g -Wall -Wextra
|
||||
|
||||
# TinyUSDZ source directory
|
||||
TINYUSDZ_DIR = ../../..
|
||||
SRC_DIR = $(TINYUSDZ_DIR)/src
|
||||
EXT_DIR = $(SRC_DIR)/external
|
||||
|
||||
# Include paths
|
||||
INCLUDES = -I$(SRC_DIR) -I$(EXT_DIR)
|
||||
|
||||
# Define to enable zstd compression
|
||||
DEFINES = -DTINYUSDZ_WITH_ZSTD_COMPRESSION -DZSTD_DISABLE_ASM=1
|
||||
|
||||
# Zstd source (amalgamated single file)
|
||||
ZSTD_SRC = $(EXT_DIR)/zstd.c
|
||||
|
||||
# TinyUSDZ sources needed for this test
|
||||
TINYUSDZ_SRCS = \
|
||||
$(SRC_DIR)/zstd-compression.cc \
|
||||
$(SRC_DIR)/tinyusdz.cc \
|
||||
$(SRC_DIR)/usda-writer.cc \
|
||||
$(SRC_DIR)/usda-reader.cc \
|
||||
$(SRC_DIR)/usdc-reader.cc \
|
||||
$(SRC_DIR)/crate-reader.cc \
|
||||
$(SRC_DIR)/ascii-parser.cc \
|
||||
$(SRC_DIR)/prim-types.cc \
|
||||
$(SRC_DIR)/value-types.cc \
|
||||
$(SRC_DIR)/pprinter.cc \
|
||||
$(SRC_DIR)/value-pprint.cc \
|
||||
$(SRC_DIR)/io-util.cc \
|
||||
$(SRC_DIR)/str-util.cc \
|
||||
$(SRC_DIR)/primvar.cc \
|
||||
$(SRC_DIR)/stage.cc \
|
||||
$(SRC_DIR)/prim-composition.cc \
|
||||
$(SRC_DIR)/composition.cc \
|
||||
$(SRC_DIR)/asset-resolution.cc \
|
||||
$(SRC_DIR)/path-util.cc \
|
||||
$(SRC_DIR)/image-loader.cc \
|
||||
$(SRC_DIR)/usdGeom.cc \
|
||||
$(SRC_DIR)/usdSkel.cc \
|
||||
$(SRC_DIR)/usdShade.cc \
|
||||
$(SRC_DIR)/usdLux.cc \
|
||||
$(SRC_DIR)/usdRender.cc \
|
||||
$(SRC_DIR)/layer.cc \
|
||||
$(SRC_DIR)/lz4-compression.cc \
|
||||
$(EXT_DIR)/lz4/lz4.c \
|
||||
$(EXT_DIR)/fpng.cpp \
|
||||
$(EXT_DIR)/string_id/database.cpp \
|
||||
$(EXT_DIR)/string_id/error.cpp \
|
||||
$(EXT_DIR)/string_id/string_id.cpp \
|
||||
$(EXT_DIR)/ryu/s2d.c \
|
||||
$(EXT_DIR)/ryu/d2s.c \
|
||||
$(EXT_DIR)/tsl/robin_hood.h
|
||||
|
||||
# Test source
|
||||
TEST_SRC = test_zstd_usd.cc
|
||||
|
||||
# Output binary
|
||||
TARGET = test_zstd_usd
|
||||
|
||||
.PHONY: all clean test
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
$(TARGET): $(TEST_SRC) $(TINYUSDZ_SRCS) $(ZSTD_SRC)
|
||||
$(CXX) $(CXXFLAGS) $(INCLUDES) $(DEFINES) -o $@ $(TEST_SRC) $(TINYUSDZ_SRCS) $(ZSTD_SRC) -lpthread
|
||||
|
||||
test: $(TARGET)
|
||||
./$(TARGET)
|
||||
|
||||
clean:
|
||||
rm -f $(TARGET) test_output.usda.zst
|
||||
51
tests/feat/zstdusd/README.md
Normal file
51
tests/feat/zstdusd/README.md
Normal file
@@ -0,0 +1,51 @@
|
||||
# Zstd USD Compression Feature Tests
|
||||
|
||||
This directory contains feature tests for zstd-compressed USD file support in TinyUSDZ.
|
||||
|
||||
## Overview
|
||||
|
||||
TinyUSDZ supports file-level zstd compression for USD files. This allows:
|
||||
- Reading zstd-compressed USDA/USDC/USDZ files (automatic detection)
|
||||
- Writing zstd-compressed USD files (via `USDWriteOptions` or `.zst` extension)
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
make
|
||||
```
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
make test
|
||||
```
|
||||
|
||||
Or directly:
|
||||
|
||||
```bash
|
||||
./test_zstd_usd
|
||||
```
|
||||
|
||||
## Test Cases
|
||||
|
||||
1. **Magic Number Detection** - Verifies zstd magic number detection
|
||||
2. **Compression Round-trip** - Basic compress/decompress cycle
|
||||
3. **Compression Levels** - Tests different compression levels (1-22)
|
||||
4. **Corrupt Data Handling** - Error handling for invalid data
|
||||
5. **GetCompressBound** - Tests the compress bound calculation
|
||||
6. **USDA Round-trip** - Full USDA compression/decompression cycle
|
||||
7. **IsZstdCompressed Wrapper** - Tests the tinyusdz namespace wrapper
|
||||
8. **Load Compressed USDA from Memory** - Tests LoadUSDFromMemory with zstd data
|
||||
9. **Memory Budget Enforcement** - Tests memory limit checking
|
||||
10. **Write Compressed USDA** - Tests SaveAsUSDA with compression
|
||||
|
||||
## Requirements
|
||||
|
||||
- C++14 compiler
|
||||
- TinyUSDZ source tree
|
||||
- Zstd library (bundled in src/external/zstd.c)
|
||||
|
||||
## Notes
|
||||
|
||||
- The tests use the amalgamated single-file zstd library
|
||||
- ZSTD_DISABLE_ASM=1 is set to avoid assembly-related linking issues
|
||||
130
tests/feat/zstdusd/compress_usda.cc
Normal file
130
tests/feat/zstdusd/compress_usda.cc
Normal file
@@ -0,0 +1,130 @@
|
||||
// Simple utility to compress a USDA file with zstd
|
||||
// Build: g++ -std=c++14 -o compress_usda compress_usda.cc -I../../../src -I../../../src/external ../../../src/external/zstd.c -DTINYUSDZ_WITH_ZSTD_COMPRESSION -DZSTD_DISABLE_ASM=1 -w
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
|
||||
#define TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
#include "zstd-compression.hh"
|
||||
|
||||
// Include zstd header for the implementation
|
||||
#include "external/zstd.h"
|
||||
|
||||
// Implement the static members
|
||||
namespace tinyusdz {
|
||||
|
||||
constexpr uint8_t ZstdCompression::kZstdMagic[4];
|
||||
|
||||
bool ZstdCompression::IsZstdCompressed(const uint8_t *data, size_t length) {
|
||||
if (!data || length < 4) {
|
||||
return false;
|
||||
}
|
||||
return (data[0] == kZstdMagic[0] && data[1] == kZstdMagic[1] &&
|
||||
data[2] == kZstdMagic[2] && data[3] == kZstdMagic[3]);
|
||||
}
|
||||
|
||||
bool ZstdCompression::Compress(const uint8_t *input, size_t inputSize,
|
||||
std::vector<uint8_t> *output,
|
||||
int compressionLevel, std::string *err) {
|
||||
if (!input || inputSize == 0) {
|
||||
if (err) *err = "Invalid input data";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!output) {
|
||||
if (err) *err = "Output buffer is null";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Clamp compression level to valid range
|
||||
if (compressionLevel < 1) compressionLevel = 1;
|
||||
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
|
||||
|
||||
// Get maximum compressed size
|
||||
size_t maxCompressedSize = ZSTD_compressBound(inputSize);
|
||||
|
||||
// Allocate output buffer
|
||||
output->resize(maxCompressedSize);
|
||||
|
||||
// Compress
|
||||
size_t result =
|
||||
ZSTD_compress(output->data(), maxCompressedSize, input, inputSize,
|
||||
compressionLevel);
|
||||
|
||||
if (ZSTD_isError(result)) {
|
||||
if (err) {
|
||||
*err = "Zstd compression failed: ";
|
||||
*err += ZSTD_getErrorName(result);
|
||||
}
|
||||
output->clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
// Shrink to actual compressed size
|
||||
output->resize(result);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t ZstdCompression::GetCompressBound(size_t inputSize) {
|
||||
return ZSTD_compressBound(inputSize);
|
||||
}
|
||||
|
||||
} // namespace tinyusdz
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 3) {
|
||||
std::cerr << "Usage: " << argv[0] << " <input.usda> <output.usda.zst> [compression_level]" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* input_file = argv[1];
|
||||
const char* output_file = argv[2];
|
||||
int compression_level = 5;
|
||||
if (argc >= 4) {
|
||||
compression_level = std::atoi(argv[3]);
|
||||
}
|
||||
|
||||
// Read input file
|
||||
std::ifstream ifs(input_file, std::ios::binary);
|
||||
if (!ifs.is_open()) {
|
||||
std::cerr << "Failed to open input file: " << input_file << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> input_data((std::istreambuf_iterator<char>(ifs)),
|
||||
std::istreambuf_iterator<char>());
|
||||
ifs.close();
|
||||
|
||||
std::cout << "Input file: " << input_file << std::endl;
|
||||
std::cout << "Input size: " << input_data.size() << " bytes" << std::endl;
|
||||
|
||||
// Compress
|
||||
std::vector<uint8_t> compressed;
|
||||
std::string err;
|
||||
bool ok = tinyusdz::ZstdCompression::Compress(
|
||||
input_data.data(), input_data.size(), &compressed, compression_level, &err);
|
||||
|
||||
if (!ok) {
|
||||
std::cerr << "Compression failed: " << err << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::cout << "Compressed size: " << compressed.size() << " bytes" << std::endl;
|
||||
std::cout << "Compression ratio: " << (100.0 * compressed.size() / input_data.size()) << "%" << std::endl;
|
||||
|
||||
// Write output file
|
||||
std::ofstream ofs(output_file, std::ios::binary);
|
||||
if (!ofs.is_open()) {
|
||||
std::cerr << "Failed to open output file: " << output_file << std::endl;
|
||||
return 1;
|
||||
}
|
||||
ofs.write(reinterpret_cast<const char*>(compressed.data()), compressed.size());
|
||||
ofs.close();
|
||||
|
||||
std::cout << "Wrote compressed file: " << output_file << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
388
tests/feat/zstdusd/test_zstd_usd.cc
Normal file
388
tests/feat/zstdusd/test_zstd_usd.cc
Normal file
@@ -0,0 +1,388 @@
|
||||
// SPDX-License-Identifier: Apache 2.0
|
||||
// Copyright 2024-Present Light Transport Entertainment Inc.
|
||||
//
|
||||
// Feature test for zstd-compressed USD file support
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <cstring>
|
||||
#include <cstdlib>
|
||||
|
||||
#define TINYUSDZ_WITH_ZSTD_COMPRESSION
|
||||
|
||||
#include "tinyusdz.hh"
|
||||
#include "usda-writer.hh"
|
||||
#include "zstd-compression.hh"
|
||||
|
||||
// Simple test macros
|
||||
#define TEST_ASSERT(cond, msg) do { \
|
||||
if (!(cond)) { \
|
||||
std::cerr << "FAILED: " << msg << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
|
||||
return 1; \
|
||||
} \
|
||||
std::cout << "PASSED: " << msg << std::endl; \
|
||||
} while(0)
|
||||
|
||||
#define TEST_ASSERT_FALSE(cond, msg) TEST_ASSERT(!(cond), msg)
|
||||
|
||||
// Test 1: Magic number detection
|
||||
int test_magic_number_detection() {
|
||||
std::cout << "\n=== Test: Magic Number Detection ===" << std::endl;
|
||||
|
||||
// Valid zstd magic number
|
||||
uint8_t valid_magic[] = {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x00, 0x00};
|
||||
TEST_ASSERT(tinyusdz::ZstdCompression::IsZstdCompressed(valid_magic, 8),
|
||||
"Should detect valid zstd magic");
|
||||
|
||||
// Invalid data
|
||||
uint8_t invalid_data[] = {0x00, 0x00, 0x00, 0x00};
|
||||
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(invalid_data, 4),
|
||||
"Should reject non-zstd data");
|
||||
|
||||
// USDA header
|
||||
uint8_t usda_header[] = {'#', 'u', 's', 'd', 'a', ' ', '1', '.'};
|
||||
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(usda_header, 8),
|
||||
"Should reject USDA header");
|
||||
|
||||
// USDC header
|
||||
uint8_t usdc_header[] = {'P', 'X', 'R', '-', 'U', 'S', 'D', 'C'};
|
||||
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(usdc_header, 8),
|
||||
"Should reject USDC header");
|
||||
|
||||
// Too short data
|
||||
uint8_t short_data[] = {0x28, 0xB5};
|
||||
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(short_data, 2),
|
||||
"Should reject too short data");
|
||||
|
||||
// Null data
|
||||
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(nullptr, 0),
|
||||
"Should reject null data");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 2: Basic compression/decompression round-trip
|
||||
int test_compression_roundtrip() {
|
||||
std::cout << "\n=== Test: Compression Round-trip ===" << std::endl;
|
||||
|
||||
const char* test_data = "This is a test string for zstd compression. "
|
||||
"It should be compressed and decompressed correctly.";
|
||||
size_t test_size = strlen(test_data);
|
||||
|
||||
std::vector<uint8_t> compressed;
|
||||
std::string err;
|
||||
|
||||
// Compress
|
||||
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||
reinterpret_cast<const uint8_t*>(test_data), test_size,
|
||||
&compressed, 5, &err);
|
||||
TEST_ASSERT(compress_ok, "Compression should succeed");
|
||||
TEST_ASSERT(compressed.size() > 0, "Compressed size should be > 0");
|
||||
TEST_ASSERT(compressed.size() < test_size, "Compressed data should be smaller");
|
||||
|
||||
// Verify magic number
|
||||
TEST_ASSERT(tinyusdz::ZstdCompression::IsZstdCompressed(compressed.data(), compressed.size()),
|
||||
"Compressed data should have zstd magic number");
|
||||
|
||||
// Get decompressed size
|
||||
size_t decompressed_size = tinyusdz::ZstdCompression::GetDecompressedSize(
|
||||
compressed.data(), compressed.size(), &err);
|
||||
TEST_ASSERT(decompressed_size == test_size, "Decompressed size should match original");
|
||||
|
||||
// Decompress
|
||||
std::vector<uint8_t> decompressed;
|
||||
bool decompress_ok = tinyusdz::ZstdCompression::Decompress(
|
||||
compressed.data(), compressed.size(), &decompressed, &err);
|
||||
TEST_ASSERT(decompress_ok, "Decompression should succeed");
|
||||
TEST_ASSERT(decompressed.size() == test_size, "Decompressed data size should match");
|
||||
TEST_ASSERT(memcmp(decompressed.data(), test_data, test_size) == 0,
|
||||
"Decompressed data should match original");
|
||||
|
||||
std::cout << "Compression ratio: " << test_size << " -> " << compressed.size()
|
||||
<< " (" << (100.0 * compressed.size() / test_size) << "%)" << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 3: Different compression levels
|
||||
int test_compression_levels() {
|
||||
std::cout << "\n=== Test: Compression Levels ===" << std::endl;
|
||||
|
||||
// Generate some compressible test data
|
||||
std::string test_data;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
test_data += "Hello World! This is test data for compression level testing. ";
|
||||
}
|
||||
|
||||
std::string err;
|
||||
size_t prev_size = test_data.size() + 1; // Start larger than input
|
||||
|
||||
for (int level = 1; level <= 9; level += 4) {
|
||||
std::vector<uint8_t> compressed;
|
||||
bool ok = tinyusdz::ZstdCompression::Compress(
|
||||
reinterpret_cast<const uint8_t*>(test_data.data()), test_data.size(),
|
||||
&compressed, level, &err);
|
||||
TEST_ASSERT(ok, "Compression at level " + std::to_string(level) + " should succeed");
|
||||
|
||||
std::cout << "Level " << level << ": " << test_data.size() << " -> "
|
||||
<< compressed.size() << " bytes ("
|
||||
<< (100.0 * compressed.size() / test_data.size()) << "%)" << std::endl;
|
||||
|
||||
// Higher levels should generally give same or better compression
|
||||
// (not strictly enforced, but good to log)
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 4: Error handling for corrupt data
|
||||
int test_corrupt_data_handling() {
|
||||
std::cout << "\n=== Test: Corrupt Data Handling ===" << std::endl;
|
||||
|
||||
std::string err;
|
||||
|
||||
// Create corrupt data with valid magic but invalid content
|
||||
uint8_t corrupt_data[] = {0x28, 0xB5, 0x2F, 0xFD, 0xFF, 0xFF, 0xFF, 0xFF};
|
||||
std::vector<uint8_t> output;
|
||||
|
||||
bool ok = tinyusdz::ZstdCompression::Decompress(corrupt_data, 8, &output, &err);
|
||||
TEST_ASSERT_FALSE(ok, "Decompression of corrupt data should fail");
|
||||
TEST_ASSERT(err.size() > 0, "Error message should be provided");
|
||||
|
||||
std::cout << "Expected error: " << err << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 5: GetCompressBound
|
||||
int test_compress_bound() {
|
||||
std::cout << "\n=== Test: GetCompressBound ===" << std::endl;
|
||||
|
||||
size_t bound = tinyusdz::ZstdCompression::GetCompressBound(1000);
|
||||
TEST_ASSERT(bound > 1000, "Compress bound should be larger than input size");
|
||||
|
||||
bound = tinyusdz::ZstdCompression::GetCompressBound(0);
|
||||
TEST_ASSERT(bound > 0, "Compress bound for 0 should still return some value");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 6: USDA file round-trip with compression
|
||||
int test_usda_roundtrip() {
|
||||
std::cout << "\n=== Test: USDA Round-trip with Compression ===" << std::endl;
|
||||
|
||||
// Create a simple stage
|
||||
tinyusdz::Stage stage;
|
||||
stage.metas().doc = "Test USD file for zstd compression";
|
||||
|
||||
// Add a simple Xform prim
|
||||
tinyusdz::Xform xform;
|
||||
xform.name = "TestXform";
|
||||
|
||||
tinyusdz::Prim prim;
|
||||
prim.set_data(xform);
|
||||
stage.root_prims().push_back(prim);
|
||||
|
||||
// Export to string first
|
||||
std::string usda_content = stage.ExportToString();
|
||||
TEST_ASSERT(usda_content.size() > 0, "USDA export should produce content");
|
||||
|
||||
// Compress the USDA content
|
||||
std::vector<uint8_t> compressed;
|
||||
std::string err;
|
||||
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||
reinterpret_cast<const uint8_t*>(usda_content.data()), usda_content.size(),
|
||||
&compressed, 5, &err);
|
||||
TEST_ASSERT(compress_ok, "USDA compression should succeed");
|
||||
|
||||
// Decompress
|
||||
std::vector<uint8_t> decompressed;
|
||||
bool decompress_ok = tinyusdz::ZstdCompression::Decompress(
|
||||
compressed.data(), compressed.size(), &decompressed, &err);
|
||||
TEST_ASSERT(decompress_ok, "USDA decompression should succeed");
|
||||
|
||||
// Verify content matches
|
||||
std::string decompressed_content(decompressed.begin(), decompressed.end());
|
||||
TEST_ASSERT(decompressed_content == usda_content,
|
||||
"Decompressed USDA content should match original");
|
||||
|
||||
std::cout << "USDA size: " << usda_content.size() << " -> " << compressed.size()
|
||||
<< " bytes (" << (100.0 * compressed.size() / usda_content.size()) << "%)"
|
||||
<< std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 7: IsZstdCompressed wrapper function in tinyusdz namespace
|
||||
int test_tinyusdz_is_zstd_compressed() {
|
||||
std::cout << "\n=== Test: tinyusdz::IsZstdCompressed ===" << std::endl;
|
||||
|
||||
// Valid zstd magic
|
||||
uint8_t valid_magic[] = {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x00, 0x00};
|
||||
TEST_ASSERT(tinyusdz::IsZstdCompressed(valid_magic, 8),
|
||||
"tinyusdz::IsZstdCompressed should detect valid zstd magic");
|
||||
|
||||
// Invalid data
|
||||
uint8_t invalid_data[] = {'#', 'u', 's', 'd', 'a'};
|
||||
TEST_ASSERT_FALSE(tinyusdz::IsZstdCompressed(invalid_data, 5),
|
||||
"tinyusdz::IsZstdCompressed should reject non-zstd data");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 8: Load zstd-compressed USDA from memory
|
||||
int test_load_compressed_usda_from_memory() {
|
||||
std::cout << "\n=== Test: Load Compressed USDA from Memory ===" << std::endl;
|
||||
|
||||
// Create a minimal USDA content
|
||||
const char* usda_content = R"(#usda 1.0
|
||||
(
|
||||
doc = "Test compressed USDA"
|
||||
)
|
||||
|
||||
def Xform "TestXform"
|
||||
{
|
||||
}
|
||||
)";
|
||||
size_t usda_size = strlen(usda_content);
|
||||
|
||||
// Compress it
|
||||
std::vector<uint8_t> compressed;
|
||||
std::string err;
|
||||
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||
reinterpret_cast<const uint8_t*>(usda_content), usda_size,
|
||||
&compressed, 5, &err);
|
||||
TEST_ASSERT(compress_ok, "Compression should succeed");
|
||||
|
||||
// Load as USD from memory
|
||||
tinyusdz::Stage stage;
|
||||
std::string warn;
|
||||
bool load_ok = tinyusdz::LoadUSDFromMemory(
|
||||
compressed.data(), compressed.size(), "", &stage, &warn, &err);
|
||||
TEST_ASSERT(load_ok, "Loading compressed USDA from memory should succeed");
|
||||
|
||||
// Verify the loaded content
|
||||
TEST_ASSERT(stage.metas().doc.value == "Test compressed USDA",
|
||||
"Document string should match");
|
||||
TEST_ASSERT(stage.root_prims().size() > 0, "Should have at least one root prim");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 9: Memory budget enforcement
|
||||
int test_memory_budget() {
|
||||
std::cout << "\n=== Test: Memory Budget Enforcement ===" << std::endl;
|
||||
|
||||
// Create a large piece of test data
|
||||
std::string large_data(10 * 1024 * 1024, 'X'); // 10MB of data
|
||||
|
||||
// Compress it
|
||||
std::vector<uint8_t> compressed;
|
||||
std::string err;
|
||||
bool compress_ok = tinyusdz::ZstdCompression::Compress(
|
||||
reinterpret_cast<const uint8_t*>(large_data.data()), large_data.size(),
|
||||
&compressed, 1, &err); // Use level 1 for speed
|
||||
TEST_ASSERT(compress_ok, "Compression should succeed");
|
||||
|
||||
// Try to load with a small memory budget
|
||||
tinyusdz::USDLoadOptions options;
|
||||
options.max_memory_limit_in_mb = 1; // Only 1MB allowed
|
||||
|
||||
tinyusdz::Stage stage;
|
||||
std::string warn;
|
||||
err.clear();
|
||||
|
||||
// This should fail because decompressed size exceeds the memory limit
|
||||
// Note: This test may not work perfectly because the compressed data
|
||||
// isn't valid USD, but it tests the memory check path
|
||||
bool load_ok = tinyusdz::LoadUSDFromMemory(
|
||||
compressed.data(), compressed.size(), "", &stage, &warn, &err, options);
|
||||
|
||||
// Should fail due to memory limit
|
||||
TEST_ASSERT_FALSE(load_ok, "Loading should fail due to memory limit");
|
||||
std::cout << "Expected error: " << err << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Test 10: Write compressed USDA file
|
||||
int test_write_compressed_usda() {
|
||||
std::cout << "\n=== Test: Write Compressed USDA File ===" << std::endl;
|
||||
|
||||
// Create a simple stage
|
||||
tinyusdz::Stage stage;
|
||||
stage.metas().doc = "Test compressed USDA write";
|
||||
|
||||
tinyusdz::Xform xform;
|
||||
xform.name = "WriteTest";
|
||||
|
||||
tinyusdz::Prim prim;
|
||||
prim.set_data(xform);
|
||||
stage.root_prims().push_back(prim);
|
||||
|
||||
// Write with compression (using options)
|
||||
tinyusdz::USDWriteOptions options;
|
||||
options.use_zstd_compression = true;
|
||||
options.zstd_compression_level = 5;
|
||||
|
||||
std::string warn, err;
|
||||
bool write_ok = tinyusdz::usda::SaveAsUSDA(
|
||||
"test_output.usda.zst", stage, &warn, &err, options);
|
||||
TEST_ASSERT(write_ok, "Writing compressed USDA should succeed");
|
||||
|
||||
// Read back the file
|
||||
std::ifstream ifs("test_output.usda.zst", std::ios::binary);
|
||||
TEST_ASSERT(ifs.is_open(), "Output file should exist");
|
||||
|
||||
std::vector<uint8_t> file_content((std::istreambuf_iterator<char>(ifs)),
|
||||
std::istreambuf_iterator<char>());
|
||||
ifs.close();
|
||||
|
||||
// Verify it's zstd compressed
|
||||
TEST_ASSERT(tinyusdz::IsZstdCompressed(file_content.data(), file_content.size()),
|
||||
"Output file should be zstd compressed");
|
||||
|
||||
// Load it back
|
||||
tinyusdz::Stage loaded_stage;
|
||||
bool load_ok = tinyusdz::LoadUSDFromMemory(
|
||||
file_content.data(), file_content.size(), "", &loaded_stage, &warn, &err);
|
||||
TEST_ASSERT(load_ok, "Loading written compressed file should succeed");
|
||||
|
||||
// Verify content
|
||||
TEST_ASSERT(loaded_stage.metas().doc.value == "Test compressed USDA write",
|
||||
"Loaded doc should match");
|
||||
|
||||
// Clean up test file
|
||||
std::remove("test_output.usda.zst");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
(void)argc;
|
||||
(void)argv;
|
||||
|
||||
std::cout << "===== Zstd USD Compression Feature Tests =====" << std::endl;
|
||||
|
||||
int failures = 0;
|
||||
|
||||
failures += test_magic_number_detection();
|
||||
failures += test_compression_roundtrip();
|
||||
failures += test_compression_levels();
|
||||
failures += test_corrupt_data_handling();
|
||||
failures += test_compress_bound();
|
||||
failures += test_usda_roundtrip();
|
||||
failures += test_tinyusdz_is_zstd_compressed();
|
||||
failures += test_load_compressed_usda_from_memory();
|
||||
failures += test_memory_budget();
|
||||
failures += test_write_compressed_usda();
|
||||
|
||||
std::cout << "\n===== Summary =====" << std::endl;
|
||||
if (failures == 0) {
|
||||
std::cout << "All tests PASSED!" << std::endl;
|
||||
return 0;
|
||||
} else {
|
||||
std::cout << failures << " test(s) FAILED!" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user