Add zstd compression support for USD files

Add file-level zstd compression/decompression for USD files (USDA, USDC, USDZ).
This allows transparent loading of zstd-compressed USD files and optional
compression when writing.

Features:
- Automatic detection by zstd magic number (0x28 0xB5 0x2F 0xFD)
- Transparent decompression in LoadUSDFromMemory()
- Optional compression in SaveAsUSDA() and SaveAsUSDCToFile()
- Auto-detection of .zst file extension for compression
- Configurable compression level (1-22, default 5)
- Memory budget enforcement during decompression
- Uses amalgamated single-file zstd library (v1.5.6)

Build options:
- TINYUSDZ_WITH_ZSTD_COMPRESSION=ON (default)
- TINYUSDZ_USE_SYSTEM_ZSTD=ON (use system library)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Syoyo Fujita
2026-01-14 12:28:43 +09:00
parent 1eb03be321
commit 4360e4d33b
16 changed files with 55326 additions and 36 deletions

View File

@@ -186,6 +186,11 @@ endif ()
option(TINYUSDZ_USE_SYSTEM_ZLIB
"Use system's zlib instead of miniz for TinyEXR/TIFF" OFF)
# -- Zstd compression --
option(TINYUSDZ_WITH_ZSTD_COMPRESSION
"Enable zstd compression for USD files" ON)
option(TINYUSDZ_USE_SYSTEM_ZSTD
"Use system's zstd instead of bundled version" OFF)
option(
TINYUSDZ_PRODUCTION_BUILD
@@ -667,6 +672,21 @@ if(TINYUSDZ_WITH_TIFF OR TINYUSDZ_WITH_EXR)
endif()
# -- Zstd compression --
if(TINYUSDZ_WITH_ZSTD_COMPRESSION)
if(NOT TINYUSDZ_USE_SYSTEM_ZSTD)
list(APPEND TINYUSDZ_DEP_SOURCES ${PROJECT_SOURCE_DIR}/src/external/zstd.c)
# Disable ASM to avoid linker issues, suppress warnings for third-party code
set_source_files_properties(
${PROJECT_SOURCE_DIR}/src/external/zstd.c
PROPERTIES COMPILE_DEFINITIONS "ZSTD_DISABLE_ASM=1"
COMPILE_FLAGS "-w")
else()
find_package(zstd REQUIRED)
endif()
list(APPEND TINYUSDZ_DEP_SOURCES ${PROJECT_SOURCE_DIR}/src/zstd-compression.cc)
endif()
if(TINYUSDZ_WITH_ALAC_AUDIO)
list(APPEND TINYUSDZ_DEP_SOURCES
${PROJECT_SOURCE_DIR}/src/external/alac/codec/EndianPortable.c
@@ -1341,6 +1361,13 @@ foreach(TINYUSDZ_LIB_TARGET ${TINYUSDZ_LIBS})
target_compile_definitions(${TINYUSDZ_LIB_TARGET} PRIVATE "TINYUSDZ_NO_STB_IMAGE_WRITE_IMPLEMENTATION")
endif()
if(TINYUSDZ_WITH_ZSTD_COMPRESSION)
target_compile_definitions(${TINYUSDZ_LIB_TARGET} PRIVATE "TINYUSDZ_WITH_ZSTD_COMPRESSION")
if(TINYUSDZ_USE_SYSTEM_ZSTD)
target_link_libraries(${TINYUSDZ_LIB_TARGET} PUBLIC zstd::zstd)
endif()
endif()
target_include_directories(${TINYUSDZ_LIB_TARGET}
PRIVATE ${PROJECT_SOURCE_DIR}/src)

BIN
models/suzanne-pbr.usda.zst Normal file

Binary file not shown.

51006
src/external/zstd.c vendored Normal file

File diff suppressed because it is too large Load Diff

3089
src/external/zstd.h vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -26,6 +26,7 @@
#include "integerCoding.h"
#include "io-util.hh"
#include "lz4-compression.hh"
#include "zstd-compression.hh"
#include "pprinter.hh"
#include "str-util.hh"
#include "stream-reader.hh"
@@ -920,6 +921,51 @@ bool LoadUSDFromMemory(const uint8_t *addr, const size_t length,
const std::string &base_dir, Stage *stage,
std::string *warn, std::string *err,
const USDLoadOptions &options) {
// Check for zstd-compressed data first (file-level compression)
if (IsZstdCompressed(addr, length)) {
DCOUT("Detected as zstd-compressed USD.");
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
// Get decompressed size for memory budget check
std::string zstd_err;
size_t decompressed_size = ZstdCompression::GetDecompressedSize(addr, length, &zstd_err);
if (decompressed_size == 0) {
if (err) {
(*err) += "Failed to get zstd decompressed size: " + zstd_err + "\n";
}
return false;
}
// Check against memory budget
size_t max_length = size_t(1024) * size_t(1024) * size_t(options.max_memory_limit_in_mb);
if (decompressed_size > max_length) {
if (err) {
(*err) += "Decompressed USD size (" + std::to_string(decompressed_size) +
" bytes) exceeds memory limit (" + std::to_string(max_length) + " bytes)\n";
}
return false;
}
// Decompress
std::vector<uint8_t> decompressed_data;
if (!ZstdCompression::Decompress(addr, length, &decompressed_data, &zstd_err)) {
if (err) {
(*err) += "Failed to decompress zstd data: " + zstd_err + "\n";
}
return false;
}
// Recursively call LoadUSDFromMemory with decompressed data
return LoadUSDFromMemory(decompressed_data.data(), decompressed_data.size(),
base_dir, stage, warn, err, options);
#else
if (err) {
(*err) += "zstd-compressed USD file detected, but zstd compression support is not enabled. "
"Rebuild with TINYUSDZ_WITH_ZSTD_COMPRESSION=ON.\n";
}
return false;
#endif
}
if (IsUSDC(addr, length)) {
DCOUT("Detected as USDC.");
return LoadUSDCFromMemory(addr, length, base_dir, stage, warn, err,
@@ -1086,6 +1132,10 @@ bool IsUSDZ(const uint8_t *addr, const size_t length) {
return ParseUSDZHeader(addr, length, /* [out] assets */ nullptr, &warn, &err);
}
bool IsZstdCompressed(const uint8_t *addr, const size_t length) {
return ZstdCompression::IsZstdCompressed(addr, length);
}
bool IsUSD(const std::string &filename, std::string *detected_format) {
if (IsUSDA(filename)) {
if (detected_format) {

View File

@@ -138,13 +138,24 @@ struct USDLoadOptions {
// - Realtime(moderate resource size limit)
// - DCC(for data conversion. Unlimited resource size)
#if 0 // TODO
//struct USDWriteOptions
//{
//
//
//};
#endif
///
/// Options for writing USD files.
///
struct USDWriteOptions {
///
/// Enable zstd compression for output file.
/// When enabled, the entire USD file is wrapped with zstd compression.
/// Also auto-detected when filename ends with ".zst" extension.
///
bool use_zstd_compression{false};
///
/// Zstd compression level (1-22).
/// Higher values give better compression but slower speed.
/// Default is 5 (good balance of speed and ratio).
///
int zstd_compression_level{5};
};
//
@@ -469,6 +480,10 @@ bool IsUSDC(const uint8_t *addr, const size_t length);
bool IsUSDZ(const std::string &filename);
bool IsUSDZ(const uint8_t *addr, const size_t length);
// Test if input is zstd-compressed data (by magic number).
// This is for file-level compression, not internal USDC LZ4 compression.
bool IsZstdCompressed(const uint8_t *addr, const size_t length);
} // namespace tinyusdz
#endif // TINYUSDZ_HH_

View File

@@ -17,46 +17,131 @@
#include "value-pprint.hh"
#include "tinyusdz.hh"
#include "io-util.hh"
#include "str-util.hh"
#include "zstd-compression.hh"
namespace tinyusdz {
namespace usda {
namespace {
// Check if filename ends with ".zst" extension (case-insensitive)
bool HasZstdExtension(const std::string &filename) {
if (filename.size() < 4) return false;
std::string ext = filename.substr(filename.size() - 4);
return (ext == ".zst" || ext == ".ZST");
}
#if defined(_WIN32)
bool HasZstdExtension(const std::wstring &filename) {
if (filename.size() < 4) return false;
std::wstring ext = filename.substr(filename.size() - 4);
return (ext == L".zst" || ext == L".ZST");
}
#endif
} // namespace
bool SaveAsUSDA(const std::string &filename, const Stage &stage,
std::string *warn, std::string *err) {
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err) {
(void)warn;
(void)err;
// TODO: Handle warn and err on export.
std::string s = stage.ExportToString();
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
if (!output) {
if (err) {
(*err) = "output parameter is null.\n";
}
return false;
}
std::cout << "Wrote to [" << filename << "]\n";
*output = stage.ExportToString();
return true;
}
bool SaveAsUSDA(const std::string &filename, const Stage &stage,
std::string *warn, std::string *err,
const USDWriteOptions &options) {
(void)warn;
// Export stage to string
std::string s = stage.ExportToString();
// Check if we should use zstd compression
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
if (use_compression) {
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
// Compress the data
std::vector<uint8_t> compressed;
if (!ZstdCompression::Compress(reinterpret_cast<const uint8_t*>(s.data()), s.size(),
&compressed, options.zstd_compression_level, err)) {
return false;
}
if (!io::WriteWholeFile(filename, compressed.data(), compressed.size(), err)) {
return false;
}
std::cout << "Wrote zstd-compressed USDA to [" << filename << "] ("
<< s.size() << " -> " << compressed.size() << " bytes)\n";
#else
if (err) {
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
}
return false;
#endif
} else {
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
return false;
}
std::cout << "Wrote USDA to [" << filename << "]\n";
}
return true;
}
#if defined(_WIN32)
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage,
std::string *warn, std::string *err) {
std::string *warn, std::string *err,
const USDWriteOptions &options) {
(void)warn;
// TODO: Handle warn and err on export.
// Export stage to string
std::string s = stage.ExportToString();
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
return false;
}
// Check if we should use zstd compression
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
std::wcout << "Wrote to [" << filename << "]\n";
if (use_compression) {
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
// Compress the data
std::vector<uint8_t> compressed;
if (!ZstdCompression::Compress(reinterpret_cast<const uint8_t*>(s.data()), s.size(),
&compressed, options.zstd_compression_level, err)) {
return false;
}
if (!io::WriteWholeFile(filename, compressed.data(), compressed.size(), err)) {
return false;
}
std::wcout << L"Wrote zstd-compressed USDA to [" << filename << L"] ("
<< s.size() << L" -> " << compressed.size() << L" bytes)\n";
#else
if (err) {
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
}
return false;
#endif
} else {
if (!io::WriteWholeFile(filename, reinterpret_cast<const unsigned char *>(s.data()), s.size(), err)) {
return false;
}
std::wcout << L"Wrote USDA to [" << filename << L"]\n";
}
return true;
}
@@ -71,9 +156,9 @@ bool SaveAsUSDA(const std::wstring &filename, const Stage &stage,
namespace tinyusdz {
namespace usda {
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err) {
(void)filename;
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err) {
(void)stage;
(void)output;
(void)warn;
if (err) {
@@ -82,7 +167,33 @@ bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *wa
return false;
}
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err,
const USDWriteOptions &options) {
(void)filename;
(void)stage;
(void)warn;
(void)options;
if (err) {
(*err) = "USDA Writer feature is disabled in this build.\n";
}
return false;
}
#if defined(_WIN32)
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err,
const USDWriteOptions &options) {
(void)filename;
(void)stage;
(void)warn;
(void)options;
if (err) {
(*err) = "USDA Writer feature is disabled in this build.\n";
}
return false;
}
#endif
} // namespace usda
} // namespace tinyusdz

View File

@@ -16,18 +16,34 @@ namespace usda {
/// Save scene as USDA(ASCII)
///
/// @param[in] filename USDA filename(UTF-8). WideChar(Unicode) represented as std::string is supported on Windows.
/// If filename ends with ".zst", zstd compression is automatically enabled.
/// @param[in] stage Stage(scene graph).
/// @param[out] warn Warning message
/// @param[out] err Error message
/// @param[in] options Write options (optional). Includes zstd compression settings.
///
/// @return true upon success.
///
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err,
const USDWriteOptions &options = USDWriteOptions());
#if defined(_WIN32)
// WideChar(UNICODE) filename version.
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err,
const USDWriteOptions &options = USDWriteOptions());
#endif
///
/// Export stage as USDA string.
///
/// @param[in] stage Stage(scene graph).
/// @param[out] output USDA string output
/// @param[out] warn Warning message
/// @param[out] err Error message
///
/// @return true upon success.
///
bool SaveAsUSDA(const std::string &filename, const Stage &stage, std::string *warn, std::string *err);
#if defined(_WIN32)
// WideChar(UNICODE) filename version.
bool SaveAsUSDA(const std::wstring &filename, const Stage &stage, std::string *warn, std::string *err);
#endif
bool ExportToUSDAString(const Stage &stage, std::string *output, std::string *warn, std::string *err);
} // namespace usda
} // namespace tinyusdz

View File

@@ -41,6 +41,7 @@
#include "crate-format.hh"
#include "io-util.hh"
#include "lz4-compression.hh"
#include "zstd-compression.hh"
#include "token-type.hh"
#include "common-macros.inc"
@@ -50,6 +51,13 @@ namespace usdc {
namespace {
// Check if filename ends with ".zst" extension (case-insensitive)
bool HasZstdExtension(const std::string &filename) {
if (filename.size() < 4) return false;
std::string ext = filename.substr(filename.size() - 4);
return (ext == ".zst" || ext == ".ZST");
}
constexpr size_t kSectionNameMaxLength = 15;
#ifdef _WIN32
@@ -480,11 +488,13 @@ class Writer {
} // namespace
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
std::string *warn, std::string *err) {
std::string *warn, std::string *err,
const USDWriteOptions &options) {
#ifdef __ANDROID__
(void)filename;
(void)stage;
(void)warn;
(void)options;
if (err) {
(*err) += "Saving USDC to a file is not supported for Android platform(at the moment).\n";
@@ -498,6 +508,30 @@ bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
return false;
}
// Check if we should use zstd compression
bool use_compression = options.use_zstd_compression || HasZstdExtension(filename);
const uint8_t *write_data = output.data();
size_t write_size = output.size();
std::vector<uint8_t> compressed;
if (use_compression) {
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
if (!ZstdCompression::Compress(output.data(), output.size(),
&compressed, options.zstd_compression_level, err)) {
return false;
}
write_data = compressed.data();
write_size = compressed.size();
std::cout << "Compressing USDC with zstd (" << output.size() << " -> " << compressed.size() << " bytes)\n";
#else
if (err) {
(*err) = "zstd compression requested but TINYUSDZ_WITH_ZSTD_COMPRESSION is not enabled.\n";
}
return false;
#endif
}
#ifdef _WIN32
#if defined(_MSC_VER) || defined(__GLIBCXX__) || defined(__clang__)
FILE *fp = nullptr;
@@ -530,9 +564,11 @@ bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
}
#endif
size_t n = fwrite(output.data(), /* size */ 1, /* count */ output.size(), fp);
if (n < output.size()) {
// TODO: Retry writing data when n < output.size()
size_t n = fwrite(write_data, /* size */ 1, /* count */ write_size, fp);
fclose(fp);
if (n < write_size) {
// TODO: Retry writing data when n < write_size
if (err) {
(*err) += "Failed to write data to a file.\n";
@@ -568,10 +604,12 @@ namespace tinyusdz {
namespace usdc {
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
std::string *warn, std::string *err) {
std::string *warn, std::string *err,
const USDWriteOptions &options) {
(void)filename;
(void)stage;
(void)warn;
(void)options;
if (err) {
(*err) = "USDC writer feature is disabled in this build.\n";

View File

@@ -19,15 +19,17 @@ namespace usdc {
///
/// Save scene as USDC(binary) to a file
///
/// @param[in] filename USDC filename
/// @param[in] filename USDC filename. If filename ends with ".zst", zstd compression is auto-enabled.
/// @param[in] stage Stage
/// @param[out] warn Warning message
/// @param[out] err Error message
/// @param[in] options Write options (optional). Includes zstd compression settings.
///
/// @return true upon success.
///
bool SaveAsUSDCToFile(const std::string &filename, const Stage &stage,
std::string *warn, std::string *err);
std::string *warn, std::string *err,
const USDWriteOptions &options = USDWriteOptions());
///
/// Save scene as USDC(binary) to a memory

175
src/zstd-compression.cc Normal file
View File

@@ -0,0 +1,175 @@
// SPDX-License-Identifier: Apache 2.0
// Copyright 2024-Present Light Transport Entertainment Inc.
//
// Zstd compression wrapper implementation for TinyUSDZ
#include "zstd-compression.hh"
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
#include <cstring>
#include <limits>
// Use amalgamated single-file zstd
#include "external/zstd.h"
namespace tinyusdz {
bool ZstdCompression::IsZstdCompressed(const uint8_t *data, size_t length) {
if (!data || length < 4) {
return false;
}
return (data[0] == kZstdMagic[0] && data[1] == kZstdMagic[1] &&
data[2] == kZstdMagic[2] && data[3] == kZstdMagic[3]);
}
size_t ZstdCompression::GetDecompressedSize(const uint8_t *compressed,
size_t compressedSize,
std::string *err) {
if (!compressed || compressedSize == 0) {
if (err) *err = "Invalid compressed data";
return 0;
}
unsigned long long frameContentSize =
ZSTD_getFrameContentSize(compressed, compressedSize);
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR) {
if (err) *err = "Not a valid zstd compressed frame";
return 0;
}
if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
if (err) *err = "Zstd frame does not contain content size";
return 0;
}
// Check for size overflow
if (frameContentSize > std::numeric_limits<size_t>::max()) {
if (err) *err = "Decompressed size exceeds size_t range";
return 0;
}
return static_cast<size_t>(frameContentSize);
}
bool ZstdCompression::Decompress(const uint8_t *compressed,
size_t compressedSize,
std::vector<uint8_t> *output,
std::string *err) {
if (!compressed || compressedSize == 0) {
if (err) *err = "Invalid compressed data";
return false;
}
if (!output) {
if (err) *err = "Output buffer is null";
return false;
}
// Get decompressed size first
size_t decompressedSize = GetDecompressedSize(compressed, compressedSize, err);
if (decompressedSize == 0) {
// err already set by GetDecompressedSize
return false;
}
// Allocate output buffer
// Note: TinyUSDZ has exceptions disabled, so we check capacity instead
output->clear();
if (decompressedSize > output->max_size()) {
if (err) *err = "Decompressed size exceeds maximum allocatable size";
return false;
}
output->resize(decompressedSize);
if (output->size() != decompressedSize) {
if (err) *err = "Failed to allocate memory for decompression";
return false;
}
// Decompress
size_t result =
ZSTD_decompress(output->data(), decompressedSize, compressed, compressedSize);
if (ZSTD_isError(result)) {
if (err) {
*err = "Zstd decompression failed: ";
*err += ZSTD_getErrorName(result);
}
output->clear();
return false;
}
// Verify decompressed size matches expected
if (result != decompressedSize) {
if (err) {
*err = "Decompressed size mismatch: expected " +
std::to_string(decompressedSize) + ", got " + std::to_string(result);
}
output->clear();
return false;
}
return true;
}
bool ZstdCompression::Compress(const uint8_t *input, size_t inputSize,
std::vector<uint8_t> *output,
int compressionLevel, std::string *err) {
if (!input || inputSize == 0) {
if (err) *err = "Invalid input data";
return false;
}
if (!output) {
if (err) *err = "Output buffer is null";
return false;
}
// Clamp compression level to valid range
if (compressionLevel < 1) compressionLevel = 1;
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
// Get maximum compressed size
size_t maxCompressedSize = ZSTD_compressBound(inputSize);
// Allocate output buffer
// Note: TinyUSDZ has exceptions disabled, so we check capacity instead
output->clear();
if (maxCompressedSize > output->max_size()) {
if (err) *err = "Compressed buffer size exceeds maximum allocatable size";
return false;
}
output->resize(maxCompressedSize);
if (output->size() != maxCompressedSize) {
if (err) *err = "Failed to allocate memory for compression";
return false;
}
// Compress
size_t result =
ZSTD_compress(output->data(), maxCompressedSize, input, inputSize,
compressionLevel);
if (ZSTD_isError(result)) {
if (err) {
*err = "Zstd compression failed: ";
*err += ZSTD_getErrorName(result);
}
output->clear();
return false;
}
// Shrink to actual compressed size
output->resize(result);
return true;
}
size_t ZstdCompression::GetCompressBound(size_t inputSize) {
return ZSTD_compressBound(inputSize);
}
} // namespace tinyusdz
#endif // TINYUSDZ_WITH_ZSTD_COMPRESSION

116
src/zstd-compression.hh Normal file
View File

@@ -0,0 +1,116 @@
// SPDX-License-Identifier: Apache 2.0
// Copyright 2024-Present Light Transport Entertainment Inc.
//
// Zstd compression wrapper for TinyUSDZ
// Provides file-level zstd compression/decompression for USD files
#pragma once
#ifndef ZSTD_COMPRESSION_HH_
#define ZSTD_COMPRESSION_HH_
#ifdef TINYUSDZ_WITH_ZSTD_COMPRESSION
#include <cstddef>
#include <cstdint>
#include <string>
#include <vector>
namespace tinyusdz {
///
/// Zstd compression utility class.
/// Provides static methods for compressing and decompressing data using zstd.
///
class ZstdCompression {
public:
///
/// Check if data starts with zstd magic number (0x28 0xB5 0x2F 0xFD)
/// @param[in] data Pointer to the data to check
/// @param[in] length Length of the data in bytes
/// @return true if data starts with zstd magic number
///
static bool IsZstdCompressed(const uint8_t *data, size_t length);
///
/// Get the decompressed size of zstd-compressed data.
/// This can be used to check memory budget before decompression.
/// @param[in] compressed Pointer to compressed data
/// @param[in] compressedSize Size of compressed data in bytes
/// @param[out] err Error message if size cannot be determined
/// @return Decompressed size in bytes, or 0 on error
///
static size_t GetDecompressedSize(const uint8_t *compressed,
size_t compressedSize, std::string *err);
///
/// Decompress zstd-compressed data.
/// @param[in] compressed Pointer to compressed data
/// @param[in] compressedSize Size of compressed data in bytes
/// @param[out] output Vector to store decompressed data
/// @param[out] err Error message on failure
/// @return true on success, false on failure
///
static bool Decompress(const uint8_t *compressed, size_t compressedSize,
std::vector<uint8_t> *output, std::string *err);
///
/// Compress data using zstd.
/// @param[in] input Pointer to input data
/// @param[in] inputSize Size of input data in bytes
/// @param[out] output Vector to store compressed data
/// @param[in] compressionLevel Compression level (1-22, default 5)
/// @param[out] err Error message on failure
/// @return true on success, false on failure
///
static bool Compress(const uint8_t *input, size_t inputSize,
std::vector<uint8_t> *output, int compressionLevel,
std::string *err);
///
/// Get the maximum compressed size for a given input size.
/// @param[in] inputSize Size of input data in bytes
/// @return Maximum possible compressed size
///
static size_t GetCompressBound(size_t inputSize);
/// Default compression level (5 is a good balance of speed and ratio)
static constexpr int kDefaultCompressionLevel = 5;
/// Zstd magic number bytes
static constexpr uint8_t kZstdMagic[4] = {0x28, 0xB5, 0x2F, 0xFD};
};
} // namespace tinyusdz
#else // !TINYUSDZ_WITH_ZSTD_COMPRESSION
// Stub implementation when zstd compression is disabled
namespace tinyusdz {
class ZstdCompression {
public:
static bool IsZstdCompressed(const uint8_t *, size_t) { return false; }
static size_t GetDecompressedSize(const uint8_t *, size_t, std::string *err) {
if (err) *err = "Zstd compression support not enabled";
return 0;
}
static bool Decompress(const uint8_t *, size_t, std::vector<uint8_t> *,
std::string *err) {
if (err) *err = "Zstd compression support not enabled";
return false;
}
static bool Compress(const uint8_t *, size_t, std::vector<uint8_t> *, int,
std::string *err) {
if (err) *err = "Zstd compression support not enabled";
return false;
}
static size_t GetCompressBound(size_t) { return 0; }
static constexpr int kDefaultCompressionLevel = 5;
};
} // namespace tinyusdz
#endif // TINYUSDZ_WITH_ZSTD_COMPRESSION
#endif // ZSTD_COMPRESSION_HH_

View File

@@ -0,0 +1,76 @@
# Makefile for zstd USD compression feature tests
# This is a standalone Makefile that builds and runs the zstd compression tests
CXX = g++
CXXFLAGS = -std=c++14 -O2 -g -Wall -Wextra
# TinyUSDZ source directory
TINYUSDZ_DIR = ../../..
SRC_DIR = $(TINYUSDZ_DIR)/src
EXT_DIR = $(SRC_DIR)/external
# Include paths
INCLUDES = -I$(SRC_DIR) -I$(EXT_DIR)
# Define to enable zstd compression
DEFINES = -DTINYUSDZ_WITH_ZSTD_COMPRESSION -DZSTD_DISABLE_ASM=1
# Zstd source (amalgamated single file)
ZSTD_SRC = $(EXT_DIR)/zstd.c
# TinyUSDZ sources needed for this test
TINYUSDZ_SRCS = \
$(SRC_DIR)/zstd-compression.cc \
$(SRC_DIR)/tinyusdz.cc \
$(SRC_DIR)/usda-writer.cc \
$(SRC_DIR)/usda-reader.cc \
$(SRC_DIR)/usdc-reader.cc \
$(SRC_DIR)/crate-reader.cc \
$(SRC_DIR)/ascii-parser.cc \
$(SRC_DIR)/prim-types.cc \
$(SRC_DIR)/value-types.cc \
$(SRC_DIR)/pprinter.cc \
$(SRC_DIR)/value-pprint.cc \
$(SRC_DIR)/io-util.cc \
$(SRC_DIR)/str-util.cc \
$(SRC_DIR)/primvar.cc \
$(SRC_DIR)/stage.cc \
$(SRC_DIR)/prim-composition.cc \
$(SRC_DIR)/composition.cc \
$(SRC_DIR)/asset-resolution.cc \
$(SRC_DIR)/path-util.cc \
$(SRC_DIR)/image-loader.cc \
$(SRC_DIR)/usdGeom.cc \
$(SRC_DIR)/usdSkel.cc \
$(SRC_DIR)/usdShade.cc \
$(SRC_DIR)/usdLux.cc \
$(SRC_DIR)/usdRender.cc \
$(SRC_DIR)/layer.cc \
$(SRC_DIR)/lz4-compression.cc \
$(EXT_DIR)/lz4/lz4.c \
$(EXT_DIR)/fpng.cpp \
$(EXT_DIR)/string_id/database.cpp \
$(EXT_DIR)/string_id/error.cpp \
$(EXT_DIR)/string_id/string_id.cpp \
$(EXT_DIR)/ryu/s2d.c \
$(EXT_DIR)/ryu/d2s.c \
$(EXT_DIR)/tsl/robin_hood.h
# Test source
TEST_SRC = test_zstd_usd.cc
# Output binary
TARGET = test_zstd_usd
.PHONY: all clean test
all: $(TARGET)
$(TARGET): $(TEST_SRC) $(TINYUSDZ_SRCS) $(ZSTD_SRC)
$(CXX) $(CXXFLAGS) $(INCLUDES) $(DEFINES) -o $@ $(TEST_SRC) $(TINYUSDZ_SRCS) $(ZSTD_SRC) -lpthread
test: $(TARGET)
./$(TARGET)
clean:
rm -f $(TARGET) test_output.usda.zst

View File

@@ -0,0 +1,51 @@
# Zstd USD Compression Feature Tests
This directory contains feature tests for zstd-compressed USD file support in TinyUSDZ.
## Overview
TinyUSDZ supports file-level zstd compression for USD files. This allows:
- Reading zstd-compressed USDA/USDC/USDZ files (automatic detection)
- Writing zstd-compressed USD files (via `USDWriteOptions` or `.zst` extension)
## Building
```bash
make
```
## Running Tests
```bash
make test
```
Or directly:
```bash
./test_zstd_usd
```
## Test Cases
1. **Magic Number Detection** - Verifies zstd magic number detection
2. **Compression Round-trip** - Basic compress/decompress cycle
3. **Compression Levels** - Tests different compression levels (1-22)
4. **Corrupt Data Handling** - Error handling for invalid data
5. **GetCompressBound** - Tests the compress bound calculation
6. **USDA Round-trip** - Full USDA compression/decompression cycle
7. **IsZstdCompressed Wrapper** - Tests the tinyusdz namespace wrapper
8. **Load Compressed USDA from Memory** - Tests LoadUSDFromMemory with zstd data
9. **Memory Budget Enforcement** - Tests memory limit checking
10. **Write Compressed USDA** - Tests SaveAsUSDA with compression
## Requirements
- C++14 compiler
- TinyUSDZ source tree
- Zstd library (bundled in src/external/zstd.c)
## Notes
- The tests use the amalgamated single-file zstd library
- ZSTD_DISABLE_ASM=1 is set to avoid assembly-related linking issues

View File

@@ -0,0 +1,130 @@
// Simple utility to compress a USDA file with zstd
// Build: g++ -std=c++14 -o compress_usda compress_usda.cc -I../../../src -I../../../src/external ../../../src/external/zstd.c -DTINYUSDZ_WITH_ZSTD_COMPRESSION -DZSTD_DISABLE_ASM=1 -w
#include <iostream>
#include <fstream>
#include <vector>
#include <cstring>
#define TINYUSDZ_WITH_ZSTD_COMPRESSION
#include "zstd-compression.hh"
// Include zstd header for the implementation
#include "external/zstd.h"
// Implement the static members
namespace tinyusdz {
constexpr uint8_t ZstdCompression::kZstdMagic[4];
bool ZstdCompression::IsZstdCompressed(const uint8_t *data, size_t length) {
if (!data || length < 4) {
return false;
}
return (data[0] == kZstdMagic[0] && data[1] == kZstdMagic[1] &&
data[2] == kZstdMagic[2] && data[3] == kZstdMagic[3]);
}
bool ZstdCompression::Compress(const uint8_t *input, size_t inputSize,
std::vector<uint8_t> *output,
int compressionLevel, std::string *err) {
if (!input || inputSize == 0) {
if (err) *err = "Invalid input data";
return false;
}
if (!output) {
if (err) *err = "Output buffer is null";
return false;
}
// Clamp compression level to valid range
if (compressionLevel < 1) compressionLevel = 1;
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
// Get maximum compressed size
size_t maxCompressedSize = ZSTD_compressBound(inputSize);
// Allocate output buffer
output->resize(maxCompressedSize);
// Compress
size_t result =
ZSTD_compress(output->data(), maxCompressedSize, input, inputSize,
compressionLevel);
if (ZSTD_isError(result)) {
if (err) {
*err = "Zstd compression failed: ";
*err += ZSTD_getErrorName(result);
}
output->clear();
return false;
}
// Shrink to actual compressed size
output->resize(result);
return true;
}
size_t ZstdCompression::GetCompressBound(size_t inputSize) {
return ZSTD_compressBound(inputSize);
}
} // namespace tinyusdz
int main(int argc, char** argv) {
if (argc < 3) {
std::cerr << "Usage: " << argv[0] << " <input.usda> <output.usda.zst> [compression_level]" << std::endl;
return 1;
}
const char* input_file = argv[1];
const char* output_file = argv[2];
int compression_level = 5;
if (argc >= 4) {
compression_level = std::atoi(argv[3]);
}
// Read input file
std::ifstream ifs(input_file, std::ios::binary);
if (!ifs.is_open()) {
std::cerr << "Failed to open input file: " << input_file << std::endl;
return 1;
}
std::vector<uint8_t> input_data((std::istreambuf_iterator<char>(ifs)),
std::istreambuf_iterator<char>());
ifs.close();
std::cout << "Input file: " << input_file << std::endl;
std::cout << "Input size: " << input_data.size() << " bytes" << std::endl;
// Compress
std::vector<uint8_t> compressed;
std::string err;
bool ok = tinyusdz::ZstdCompression::Compress(
input_data.data(), input_data.size(), &compressed, compression_level, &err);
if (!ok) {
std::cerr << "Compression failed: " << err << std::endl;
return 1;
}
std::cout << "Compressed size: " << compressed.size() << " bytes" << std::endl;
std::cout << "Compression ratio: " << (100.0 * compressed.size() / input_data.size()) << "%" << std::endl;
// Write output file
std::ofstream ofs(output_file, std::ios::binary);
if (!ofs.is_open()) {
std::cerr << "Failed to open output file: " << output_file << std::endl;
return 1;
}
ofs.write(reinterpret_cast<const char*>(compressed.data()), compressed.size());
ofs.close();
std::cout << "Wrote compressed file: " << output_file << std::endl;
return 0;
}

View File

@@ -0,0 +1,388 @@
// SPDX-License-Identifier: Apache 2.0
// Copyright 2024-Present Light Transport Entertainment Inc.
//
// Feature test for zstd-compressed USD file support
#include <iostream>
#include <fstream>
#include <cstring>
#include <cstdlib>
#define TINYUSDZ_WITH_ZSTD_COMPRESSION
#include "tinyusdz.hh"
#include "usda-writer.hh"
#include "zstd-compression.hh"
// Simple test macros
#define TEST_ASSERT(cond, msg) do { \
if (!(cond)) { \
std::cerr << "FAILED: " << msg << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
return 1; \
} \
std::cout << "PASSED: " << msg << std::endl; \
} while(0)
#define TEST_ASSERT_FALSE(cond, msg) TEST_ASSERT(!(cond), msg)
// Test 1: Magic number detection
int test_magic_number_detection() {
std::cout << "\n=== Test: Magic Number Detection ===" << std::endl;
// Valid zstd magic number
uint8_t valid_magic[] = {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x00, 0x00};
TEST_ASSERT(tinyusdz::ZstdCompression::IsZstdCompressed(valid_magic, 8),
"Should detect valid zstd magic");
// Invalid data
uint8_t invalid_data[] = {0x00, 0x00, 0x00, 0x00};
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(invalid_data, 4),
"Should reject non-zstd data");
// USDA header
uint8_t usda_header[] = {'#', 'u', 's', 'd', 'a', ' ', '1', '.'};
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(usda_header, 8),
"Should reject USDA header");
// USDC header
uint8_t usdc_header[] = {'P', 'X', 'R', '-', 'U', 'S', 'D', 'C'};
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(usdc_header, 8),
"Should reject USDC header");
// Too short data
uint8_t short_data[] = {0x28, 0xB5};
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(short_data, 2),
"Should reject too short data");
// Null data
TEST_ASSERT_FALSE(tinyusdz::ZstdCompression::IsZstdCompressed(nullptr, 0),
"Should reject null data");
return 0;
}
// Test 2: Basic compression/decompression round-trip
int test_compression_roundtrip() {
std::cout << "\n=== Test: Compression Round-trip ===" << std::endl;
const char* test_data = "This is a test string for zstd compression. "
"It should be compressed and decompressed correctly.";
size_t test_size = strlen(test_data);
std::vector<uint8_t> compressed;
std::string err;
// Compress
bool compress_ok = tinyusdz::ZstdCompression::Compress(
reinterpret_cast<const uint8_t*>(test_data), test_size,
&compressed, 5, &err);
TEST_ASSERT(compress_ok, "Compression should succeed");
TEST_ASSERT(compressed.size() > 0, "Compressed size should be > 0");
TEST_ASSERT(compressed.size() < test_size, "Compressed data should be smaller");
// Verify magic number
TEST_ASSERT(tinyusdz::ZstdCompression::IsZstdCompressed(compressed.data(), compressed.size()),
"Compressed data should have zstd magic number");
// Get decompressed size
size_t decompressed_size = tinyusdz::ZstdCompression::GetDecompressedSize(
compressed.data(), compressed.size(), &err);
TEST_ASSERT(decompressed_size == test_size, "Decompressed size should match original");
// Decompress
std::vector<uint8_t> decompressed;
bool decompress_ok = tinyusdz::ZstdCompression::Decompress(
compressed.data(), compressed.size(), &decompressed, &err);
TEST_ASSERT(decompress_ok, "Decompression should succeed");
TEST_ASSERT(decompressed.size() == test_size, "Decompressed data size should match");
TEST_ASSERT(memcmp(decompressed.data(), test_data, test_size) == 0,
"Decompressed data should match original");
std::cout << "Compression ratio: " << test_size << " -> " << compressed.size()
<< " (" << (100.0 * compressed.size() / test_size) << "%)" << std::endl;
return 0;
}
// Test 3: Different compression levels
int test_compression_levels() {
std::cout << "\n=== Test: Compression Levels ===" << std::endl;
// Generate some compressible test data
std::string test_data;
for (int i = 0; i < 1000; i++) {
test_data += "Hello World! This is test data for compression level testing. ";
}
std::string err;
size_t prev_size = test_data.size() + 1; // Start larger than input
for (int level = 1; level <= 9; level += 4) {
std::vector<uint8_t> compressed;
bool ok = tinyusdz::ZstdCompression::Compress(
reinterpret_cast<const uint8_t*>(test_data.data()), test_data.size(),
&compressed, level, &err);
TEST_ASSERT(ok, "Compression at level " + std::to_string(level) + " should succeed");
std::cout << "Level " << level << ": " << test_data.size() << " -> "
<< compressed.size() << " bytes ("
<< (100.0 * compressed.size() / test_data.size()) << "%)" << std::endl;
// Higher levels should generally give same or better compression
// (not strictly enforced, but good to log)
}
return 0;
}
// Test 4: Error handling for corrupt data
int test_corrupt_data_handling() {
std::cout << "\n=== Test: Corrupt Data Handling ===" << std::endl;
std::string err;
// Create corrupt data with valid magic but invalid content
uint8_t corrupt_data[] = {0x28, 0xB5, 0x2F, 0xFD, 0xFF, 0xFF, 0xFF, 0xFF};
std::vector<uint8_t> output;
bool ok = tinyusdz::ZstdCompression::Decompress(corrupt_data, 8, &output, &err);
TEST_ASSERT_FALSE(ok, "Decompression of corrupt data should fail");
TEST_ASSERT(err.size() > 0, "Error message should be provided");
std::cout << "Expected error: " << err << std::endl;
return 0;
}
// Test 5: GetCompressBound
int test_compress_bound() {
std::cout << "\n=== Test: GetCompressBound ===" << std::endl;
size_t bound = tinyusdz::ZstdCompression::GetCompressBound(1000);
TEST_ASSERT(bound > 1000, "Compress bound should be larger than input size");
bound = tinyusdz::ZstdCompression::GetCompressBound(0);
TEST_ASSERT(bound > 0, "Compress bound for 0 should still return some value");
return 0;
}
// Test 6: USDA file round-trip with compression
int test_usda_roundtrip() {
std::cout << "\n=== Test: USDA Round-trip with Compression ===" << std::endl;
// Create a simple stage
tinyusdz::Stage stage;
stage.metas().doc = "Test USD file for zstd compression";
// Add a simple Xform prim
tinyusdz::Xform xform;
xform.name = "TestXform";
tinyusdz::Prim prim;
prim.set_data(xform);
stage.root_prims().push_back(prim);
// Export to string first
std::string usda_content = stage.ExportToString();
TEST_ASSERT(usda_content.size() > 0, "USDA export should produce content");
// Compress the USDA content
std::vector<uint8_t> compressed;
std::string err;
bool compress_ok = tinyusdz::ZstdCompression::Compress(
reinterpret_cast<const uint8_t*>(usda_content.data()), usda_content.size(),
&compressed, 5, &err);
TEST_ASSERT(compress_ok, "USDA compression should succeed");
// Decompress
std::vector<uint8_t> decompressed;
bool decompress_ok = tinyusdz::ZstdCompression::Decompress(
compressed.data(), compressed.size(), &decompressed, &err);
TEST_ASSERT(decompress_ok, "USDA decompression should succeed");
// Verify content matches
std::string decompressed_content(decompressed.begin(), decompressed.end());
TEST_ASSERT(decompressed_content == usda_content,
"Decompressed USDA content should match original");
std::cout << "USDA size: " << usda_content.size() << " -> " << compressed.size()
<< " bytes (" << (100.0 * compressed.size() / usda_content.size()) << "%)"
<< std::endl;
return 0;
}
// Test 7: IsZstdCompressed wrapper function in tinyusdz namespace
int test_tinyusdz_is_zstd_compressed() {
std::cout << "\n=== Test: tinyusdz::IsZstdCompressed ===" << std::endl;
// Valid zstd magic
uint8_t valid_magic[] = {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x00, 0x00};
TEST_ASSERT(tinyusdz::IsZstdCompressed(valid_magic, 8),
"tinyusdz::IsZstdCompressed should detect valid zstd magic");
// Invalid data
uint8_t invalid_data[] = {'#', 'u', 's', 'd', 'a'};
TEST_ASSERT_FALSE(tinyusdz::IsZstdCompressed(invalid_data, 5),
"tinyusdz::IsZstdCompressed should reject non-zstd data");
return 0;
}
// Test 8: Load zstd-compressed USDA from memory
int test_load_compressed_usda_from_memory() {
std::cout << "\n=== Test: Load Compressed USDA from Memory ===" << std::endl;
// Create a minimal USDA content
const char* usda_content = R"(#usda 1.0
(
doc = "Test compressed USDA"
)
def Xform "TestXform"
{
}
)";
size_t usda_size = strlen(usda_content);
// Compress it
std::vector<uint8_t> compressed;
std::string err;
bool compress_ok = tinyusdz::ZstdCompression::Compress(
reinterpret_cast<const uint8_t*>(usda_content), usda_size,
&compressed, 5, &err);
TEST_ASSERT(compress_ok, "Compression should succeed");
// Load as USD from memory
tinyusdz::Stage stage;
std::string warn;
bool load_ok = tinyusdz::LoadUSDFromMemory(
compressed.data(), compressed.size(), "", &stage, &warn, &err);
TEST_ASSERT(load_ok, "Loading compressed USDA from memory should succeed");
// Verify the loaded content
TEST_ASSERT(stage.metas().doc.value == "Test compressed USDA",
"Document string should match");
TEST_ASSERT(stage.root_prims().size() > 0, "Should have at least one root prim");
return 0;
}
// Test 9: Memory budget enforcement
int test_memory_budget() {
std::cout << "\n=== Test: Memory Budget Enforcement ===" << std::endl;
// Create a large piece of test data
std::string large_data(10 * 1024 * 1024, 'X'); // 10MB of data
// Compress it
std::vector<uint8_t> compressed;
std::string err;
bool compress_ok = tinyusdz::ZstdCompression::Compress(
reinterpret_cast<const uint8_t*>(large_data.data()), large_data.size(),
&compressed, 1, &err); // Use level 1 for speed
TEST_ASSERT(compress_ok, "Compression should succeed");
// Try to load with a small memory budget
tinyusdz::USDLoadOptions options;
options.max_memory_limit_in_mb = 1; // Only 1MB allowed
tinyusdz::Stage stage;
std::string warn;
err.clear();
// This should fail because decompressed size exceeds the memory limit
// Note: This test may not work perfectly because the compressed data
// isn't valid USD, but it tests the memory check path
bool load_ok = tinyusdz::LoadUSDFromMemory(
compressed.data(), compressed.size(), "", &stage, &warn, &err, options);
// Should fail due to memory limit
TEST_ASSERT_FALSE(load_ok, "Loading should fail due to memory limit");
std::cout << "Expected error: " << err << std::endl;
return 0;
}
// Test 10: Write compressed USDA file
int test_write_compressed_usda() {
std::cout << "\n=== Test: Write Compressed USDA File ===" << std::endl;
// Create a simple stage
tinyusdz::Stage stage;
stage.metas().doc = "Test compressed USDA write";
tinyusdz::Xform xform;
xform.name = "WriteTest";
tinyusdz::Prim prim;
prim.set_data(xform);
stage.root_prims().push_back(prim);
// Write with compression (using options)
tinyusdz::USDWriteOptions options;
options.use_zstd_compression = true;
options.zstd_compression_level = 5;
std::string warn, err;
bool write_ok = tinyusdz::usda::SaveAsUSDA(
"test_output.usda.zst", stage, &warn, &err, options);
TEST_ASSERT(write_ok, "Writing compressed USDA should succeed");
// Read back the file
std::ifstream ifs("test_output.usda.zst", std::ios::binary);
TEST_ASSERT(ifs.is_open(), "Output file should exist");
std::vector<uint8_t> file_content((std::istreambuf_iterator<char>(ifs)),
std::istreambuf_iterator<char>());
ifs.close();
// Verify it's zstd compressed
TEST_ASSERT(tinyusdz::IsZstdCompressed(file_content.data(), file_content.size()),
"Output file should be zstd compressed");
// Load it back
tinyusdz::Stage loaded_stage;
bool load_ok = tinyusdz::LoadUSDFromMemory(
file_content.data(), file_content.size(), "", &loaded_stage, &warn, &err);
TEST_ASSERT(load_ok, "Loading written compressed file should succeed");
// Verify content
TEST_ASSERT(loaded_stage.metas().doc.value == "Test compressed USDA write",
"Loaded doc should match");
// Clean up test file
std::remove("test_output.usda.zst");
return 0;
}
int main(int argc, char** argv) {
(void)argc;
(void)argv;
std::cout << "===== Zstd USD Compression Feature Tests =====" << std::endl;
int failures = 0;
failures += test_magic_number_detection();
failures += test_compression_roundtrip();
failures += test_compression_levels();
failures += test_corrupt_data_handling();
failures += test_compress_bound();
failures += test_usda_roundtrip();
failures += test_tinyusdz_is_zstd_compressed();
failures += test_load_compressed_usda_from_memory();
failures += test_memory_budget();
failures += test_write_compressed_usda();
std::cout << "\n===== Summary =====" << std::endl;
if (failures == 0) {
std::cout << "All tests PASSED!" << std::endl;
return 0;
} else {
std::cout << failures << " test(s) FAILED!" << std::endl;
return 1;
}
}