diff --git a/tests/cts b/tests/cts index cc1d79f64..dca1522ba 160000 --- a/tests/cts +++ b/tests/cts @@ -1 +1 @@ -Subproject commit cc1d79f64861f6eefecf968e3adf6e20890714eb +Subproject commit dca1522bae8e512c38119e277c853c3f6436b8d6 diff --git a/tools/ktx/command.h b/tools/ktx/command.h index 01f0b1a7c..c99bbccca 100644 --- a/tools/ktx/command.h +++ b/tools/ktx/command.h @@ -10,6 +10,7 @@ #include #include #include "utility.h" +#include "platform_utils.h" #include #include @@ -312,11 +313,82 @@ struct OptionsMultiInSingleOut { ("stdin", "Use stdin as the first input file. (Using a single dash '-' as the first input file has the same effect)") ("stdout", "Use stdout as the output file. (Using a single dash '-' as the output file has the same effect)") ("files", "Input/output files. Last file specified will be used as output." - " Using a single dash '-' as an input or output file will use stdin/stdout.", cxxopts::value>(), ""); + " Using a single dash '-' as an input or output file will use stdin/stdout." + " A filepath prefixed with @ is read as a file name listing file. Listing text" + " files specify which actual files to process, one file name per line. Names can" + " be absolute paths or relative to the current directory when the application is" + " run. If the file is prefixed with @b @@ the names must be relative to the listing file.", + cxxopts::value>(), ""); opts.parse_positional("files"); opts.positional_help(" "); } + void loadFileList(const std::string &f, bool relativize, + std::vector& files, + Reporter& report) { + std::string listName(f); + listName.erase(0, relativize ? 2 : 1); + + FILE *lf = nullptr; + lf = fopenUTF8(listName, "r"); + if (!lf) { + report.fatal(rc::RUNTIME_ERROR, "Opening filename list: \"{}\" failed: {}\n", + listName.c_str(), errnoMessage()); + } + + std::string dirname; + + if (relativize) { + size_t dirnameEnd = listName.find_last_of('/'); + if (dirnameEnd == std::string::npos) { + relativize = false; + } else { + dirname = listName.substr(0, dirnameEnd + 1); + } + } + + for (;;) { + // Cross platform PATH_MAX def is too much trouble! + char buf[4096]; + buf[0] = '\0'; + + char *p = fgets(buf, sizeof(buf), lf); + if (!p) { + if (ferror(lf)) { + report.fatal(rc::RUNTIME_ERROR, "Reading filename list: \"{}\" failed: {}\n", + listName.c_str(), errnoMessage()); + fclose(lf); + } else + break; + } + + std::string readFilename(p); + while (readFilename.size()) { + if (readFilename[0] == ' ') + readFilename.erase(0, 1); + else + break; + } + + while (readFilename.size()) { + const char c = readFilename.back(); + if ((c == ' ') || (c == '\n') || (c == '\r')) + readFilename.erase(readFilename.size() - 1, 1); + else + break; + } + + if (readFilename.size()) { + if (relativize) + files.push_back(dirname + readFilename); + else + files.push_back(readFilename); + } + } + fclose(lf); + + } + void process(cxxopts::Options&, cxxopts::ParseResult& args, Reporter& report) { std::vector files; if (args.count("stdin")) @@ -334,7 +406,14 @@ struct OptionsMultiInSingleOut { outputFilepath = std::move(files.back()); files.pop_back(); - inputFilepaths = std::move(files); + std::vector::const_iterator fit; + for (fit = files.begin(); fit < files.end(); fit++) { + if (fit[0][0] == '@') { + loadFileList(*fit, fit[0][1] == '@', inputFilepaths, report); + } else { + inputFilepaths.push_back(std::move(*fit)); + } + } if (std::count(inputFilepaths.begin(), inputFilepaths.end(), "-") > 1) report.fatal_usage("'-' or --stdin as input file was specified more than once."); @@ -393,6 +472,14 @@ protected: public: OutputStream(const std::string& filepath, Reporter& report); +#if defined(__cpp_lib_char8_t) + // This is a simplest way to make this work when compiled with >= c++20 + // and the caller of this is passing the output of std::filesystem::path::u8string(). + // At some point we should consider making filepath a u8string. + OutputStream::OutputStream(const std::u8string& filepath, Reporter& report) : + OutputStream(from_u8string(filepath), report) { } +#endif + ~OutputStream(); const std::string& str() { diff --git a/tools/ktx/command_convert.cpp b/tools/ktx/command_convert.cpp index 6d38acbe4..190db0000 100644 --- a/tools/ktx/command_convert.cpp +++ b/tools/ktx/command_convert.cpp @@ -81,6 +81,10 @@ class OutputStreamEx : public OutputStream { public: OutputStreamEx(const std::string& filepath, Reporter& report) : OutputStream(filepath, report) { } +#if defined(__cpp_lib_char8_t) + OutputStreamEx(const std::u8string& filepath, Reporter& report) + : OutputStream(filepath, report) { } +#endif void writeKTX2(ktxTexture1* texture, Reporter& report) { const auto ret = ktxTexture1_WriteKTX2ToStdioStream(texture, file); diff --git a/tools/ktx/command_create.cpp b/tools/ktx/command_create.cpp index 29457ee3a..f8360f743 100644 --- a/tools/ktx/command_create.cpp +++ b/tools/ktx/command_create.cpp @@ -848,6 +848,12 @@ Create a KTX2 file from various input files. If the @e input-file is '-' the file will be read from the stdin. If the @e output-path is '-' the output file will be written to the stdout. + An @e input-file prefixed with @b \@ will be read as a file name listing file. + Listing text files specify which actual files to process, one file name per line. + Names can be absolute paths or relative to the current directory when the + application is run. If the file is prefixed with @b \@\@ the names must be + relative to the listing file. + Each @e input-file must be a valid EXR (.exr), PNG (.png) or Raw (.raw) file. PNG files with luminance (L) or luminance + alpha (LA) data will be converted to RGB as LLL and RGBA as LLLA before processing further. diff --git a/utils/platform_utils.h b/utils/platform_utils.h index c5e2fb32e..751d6a8ad 100644 --- a/utils/platform_utils.h +++ b/utils/platform_utils.h @@ -21,6 +21,31 @@ #include #endif +/* + * @internal + * @file + * @~English + * + * @brief Cross-platform utilities for handling utf-8 file names. + * + * To display UTF-8 strings streamed to the console correctly on WindowsPowerShell + * or Command Prompt they must be set to display UTF-8 text. For PowerShell run the + * following command before executing the program or add it to your $PROFILE: + * $OutputEncoding = [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new() + * See https://stackoverflow.com/questions/57131654/using-utf-8-encoding-chcp-65001-in-command-prompt-windows-powershell-window + * for more details and how to change the encoding for Command Prompt. + * + * Note that the PS console spawned by Visual Studio when running a console + * application does not load $PROFILE so displayed utf-8 will be mojibake. + * + * Note also that fmt::print works correctly without changing the console + * encoding because it uses the Windows wide char APIs to write to the + * console. + * + * @author Daniel Rákos + * @author Mark Callow + */ + #if defined(_WIN32) && !defined(_UNICODE) // For Windows, we convert the UTF-8 path to a UTF-16 path to force using // the APIs that correctly handle unicode characters. @@ -46,13 +71,22 @@ inline void InitUTF8CLI(int& argc, char* argv[]) { #if defined(_WIN32) // Windows does not support UTF-8 argv so we have to manually acquire it static std::vector> utf8Argv(argc); + // argc may be different from wargc if the caller of this is the + // secondary receiver of the command line args, e.g. in a gtest program + // where gtest removes its own args first. + int wargc; LPWSTR commandLine = GetCommandLineW(); - LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &argc); + LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &wargc); for (int i = 0; i < argc; ++i) { int byteSize = WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr); utf8Argv[i] = std::make_unique(byteSize); - WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), byteSize, nullptr, nullptr); + WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), + byteSize, nullptr, nullptr); argv[i] = utf8Argv[i].get(); + if (i == 0) { + // Skip over the removed args. + i += (wargc - argc); + } } #else // Nothing to do for other platforms @@ -79,3 +113,86 @@ inline int unlinkUTF8(const std::string& path) { return unlink(path.c_str()); #endif } + +#if defined(__cpp_lib_char8_t) + // Casting from u8string to string is not allowed in C++20. Neither + // can char8_t or std::u8string be streamed to ostreams. This provides + // an explicit conversion. Note that this does not perform any encoding. + inline std::string from_u8string(const std::u8string& s) { + return std::string(s.begin(), s.end()); + } + inline std::string to_u8string(const std::string& s) { + return std::u8string(s.begin(), s.end()); + } + + #if defined(_WIN32) && !defined(_UNICODE) + // For Windows, we convert the UTF-8 path to a UTF-16 path to force using + // the APIs that correctly handle unicode characters. + inline std::wstring DecodeUTF8Path(std::u8string u8path) { + std::wstring result; + std::string path = from_u8string(u8path); + int len = + MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast(path.length()), + NULL, 0); + if (len > 0) { + result.resize(len); + MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast(path.length()), + &result[0], len); + } + return result; + } + #else + // For other platforms convert to a regular string. + inline std::string DecodeUTF8Path(std::u8string path) { return from_u8string(path); } + #endif + + inline void InitUTF8CLI(int& argc, char* argv[], std::vector& u8argv) { + u8argv.resize(argc); + #if defined(_WIN32) + // Windows does not support UTF-8 argv so we have to manually acquire it + (void)argv; // Unused + // See note in non-char8_t InitUTF8CLI about argc vs wargc. + int wargc; + LPWSTR commandLine = GetCommandLineW(); + LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &wargc); + for (int i = 0; i < argc; ++i) { + int byteSize = + WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr); + byteSize--; // Returned byteSize includes the terminating NUL. + u8argv[i].resize(byteSize); + WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, (LPSTR)u8argv[i].data(), + byteSize, nullptr, nullptr); + if (i == 0) { + // Skip over the removed args. + i += (wargc - argc); + } + } + #else + for (int i = 0; i < argc; ++i) { + u8argv[i] = std::u8string(reinterpret_cast(argv[i])); + } + #endif + } + + inline FILE* fopenUTF8(const std::u8string& path, const std::string& mode) { + #if defined(_WIN32) + FILE* fp; + // Returned errno_t value is also set in the global errno. + (void)_wfopen_s(&fp, DecodeUTF8Path(path).c_str(), DecodeUTF8Path(mode).c_str()); + return fp; + #else + return fopen(from_u8string(path).c_str(), mode.c_str()); + #endif + } + + inline int unlinkUTF8(const std::u8string& path) { + #if defined(_WIN32) + return _wunlink(DecodeUTF8Path(path).c_str()); + #else + return unlink(DecodeUTF8Path(path).c_str()); + #endif + } +#else + #define from_u8string(s) (s) + #define to_u8string(s) (s) +#endif