Mirrors/rapidyaml
0
0
Fork 0
mirror of https://github.com/biojppm/rapidyaml.git synced 2026-01-18 21:41:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

[fix] re #205: add missing escaped characters in dquo scalars

Browse Source
This commit is contained in:
Joao Paulo Magalhaes
2022-01-24 17:19:59 +00:00
parent 07c13e27cf
commit 835bbddd9e
20 changed files with 588 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/setenv.sh vendored
View File

@@ -320,7 +320,9 @@ function c4_cfg_test()
;;
em++)
emcmake cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \
-DCMAKE_BUILD_TYPE=$BT $CMFLAGS -DCMAKE_CXX_FLAGS="-s DISABLE_EXCEPTION_CATCHING=0"
-DCMAKE_BUILD_TYPE=$BT $CMFLAGS \
-DCMAKE_CXX_FLAGS="-s DISABLE_EXCEPTION_CATCHING=0" \
-DRYML_TEST_TOOLS=OFF
;;
*)
echo "unknown compiler"

4
changelog/current.md
View File

@@ -94,13 +94,13 @@ As part of the [new feature to track source locations](https://github.com/biojpp
### Fixes
- Fix [#205](https://github.com/biojppm/rapidyaml/issues/205): add missing escape for `\b\f\0` ([PR#206](https://github.com/biojppm/rapidyaml/pulls/206)).
- Fix [#205](https://github.com/biojppm/rapidyaml/issues/205): fix parsing of escaped characters in double-quoted strings: `"\\\"\n\r\t\<TAB>\/\<SPC>\0\b\f\a\v\e\_\N\L\P"` ([PR#207](https://github.com/biojppm/rapidyaml/pulls/207)).
- Fix [#204](https://github.com/biojppm/rapidyaml/issues/204): add decoding of unicode codepoints `\x` `\u` `\U` in double-quoted scalars:
```c++
Tree tree = parse_in_arena(R"(["\u263A \xE2\x98\xBA \u2705 \U0001D11E"])");
assert(tree[0].val() == "☺ ☺ ✅ 𝄞");
```
This is mandated by the YAML standard and was missing from ryml ([PR#206](https://github.com/biojppm/rapidyaml/pulls/206)).
This is mandated by the YAML standard and was missing from ryml ([PR#207](https://github.com/biojppm/rapidyaml/pulls/207)).
- Fix [#193](https://github.com/biojppm/rapidyaml/issues/193): amalgamated header missing `#include <stdarg.h>` which prevented compilation in bare-metal `arm-none-eabi` ([PR #195](https://github.com/biojppm/rapidyaml/pull/195), requiring also [c4core #64](https://github.com/biojppm/c4core/pull/64)).
- Accept `infinity`,`inf` and `nan` as special float values (but not mixed case: eg `InFiNiTy` or `Inf` or `NaN` are not accepted) ([PR #186](https://github.com/biojppm/rapidyaml/pull/186)).
- Accept special float values with upper or mixed case: `.Inf`, `.INF`, `.NaN`, `.NAN`. Previously, only low-case `.inf` and `.nan` were accepted ([PR #186](https://github.com/biojppm/rapidyaml/pull/186)).

2
setup.py
View File

@@ -19,7 +19,7 @@ PYTHON_DIR = "api/python"
def get_readme_for_python():
with open(TOP_DIR / "README.md", "r") as fh:
with open(TOP_DIR / "README.md", "r", encoding="utf8") as fh:
marker = "<!-- endpythonreadme -->" # get everything up to this tag
return fh.read().split(marker)[0]

2
src/c4/yml/detail/checks.hpp
View File

@@ -180,7 +180,7 @@ inline void check_free_list(Tree const& t)
inline void check_arena(Tree const& t)
{
C4_CHECK(t.m_arena.len == 0 || (t.m_arena_pos >= 0 && t.m_arena_pos < t.m_arena.len));
C4_CHECK(t.m_arena.len == 0 || (t.m_arena_pos >= 0 && t.m_arena_pos <= t.m_arena.len));
C4_CHECK(t.arena_size() == t.m_arena_pos);
C4_CHECK(t.arena_slack() + t.m_arena_pos == t.m_arena.len);
}

55
src/c4/yml/detail/parser_dbg.hpp
View File

@@ -43,23 +43,68 @@
#endif
#define _c4prsp(sp) ((int)(sp).len), (sp).str
#define _c4presc(s) __c4presc(s.str, s.len)
#define _c4prc(c) (__c4prc(c) ? 2 : 1), (__c4prc(c) ? __c4prc(c) : &c)
#define _c4presc(s) __c4presc(s.str, s.len)
inline const char *__c4prc(const char &c)
{
switch(c)
{
case '\n': return "\\n";
case '\t': return "\\t";
case '\0': return "\\0";
case '\r': return "\\r";
case '\t': return "\\t";
case '\n': return "\\n";
case '\f': return "\\f";
case '\b': return "\\b";
case '\v': return "\\v";
case '\a': return "\\a";
default: return nullptr;
};
}
}
inline void __c4presc(const char *s, size_t len)
{
size_t prev = 0;
for(size_t i = 0; i < len; ++i)
printf("%.*s", _c4prc(s[i]));
{
switch(s[i])
{
case '\n' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('n'); putchar('\n'); prev = i+1; break;
case '\t' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('t'); prev = i+1; break;
case '\0' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('0'); prev = i+1; break;
case '\r' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('r'); prev = i+1; break;
case '\f' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('f'); prev = i+1; break;
case '\b' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('b'); prev = i+1; break;
case '\v' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('v'); prev = i+1; break;
case '\a' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('a'); prev = i+1; break;
case '\x1b': fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('e'); prev = i+1; break;
case -0x3e/*0xc2u*/:
if(i+1 < len)
{
if(s[i+1] == -0x60/*0xa0u*/)
{
fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('_'); prev = i+2; ++i;
}
else if(s[i+1] == -0x7b/*0x85u*/)
{
fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('N'); prev = i+2; ++i;
}
break;
}
case -0x1e/*0xe2u*/:
if(i+2 < len && s[i+1] == -0x80/*0x80u*/)
{
if(s[i+2] == -0x58/*0xa8u*/)
{
fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('L'); prev = i+3; i += 2;
}
else if(s[i+2] == -0x57/*0xa9u*/)
{
fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('P'); prev = i+3; i += 2;
}
break;
}
}
}
fwrite(s + prev, 1, len - prev, stdout);
}
#pragma clang diagnostic pop

40
src/c4/yml/emit.def.hpp
View File

@@ -268,13 +268,9 @@ template<class Writer>
void Emitter<Writer>::_write_json(NodeScalar const& sc, NodeType flags)
{
if(C4_UNLIKELY( ! sc.tag.empty()))
{
c4::yml::error("JSON does not have tags");
}
if(C4_UNLIKELY(flags.has_anchor()))
{
c4::yml::error("JSON does not have anchors");
}
_write_scalar_json(sc.scalar, flags.has_key(), flags.is_quoted());
}
@@ -282,12 +278,9 @@ template<class Writer>
void Emitter<Writer>::_write_scalar_block(csubstr s, size_t ilevel, bool explicit_key)
{
#define _rymlindent_nextline() for(size_t lv = 0; lv < ilevel+1; ++lv) { this->Writer::_do_write(" "); }
#define _ryml_add_newline() do { while(s[pos] == '\r') { this->Writer::_do_write('\r'); ++pos; RYML_ASSERT(pos <= s.len); } this->Writer::_do_write('\n'); ++pos; RYML_ASSERT(pos <= s.len); } while(0)
if(explicit_key)
this->Writer::_do_write("? ");
csubstr trimmed = s.trimr("\r\n");
csubstr trimmed = s.trimr("\n\r");
size_t numnewlines_at_end = s.len - trimmed.len - s.sub(trimmed.len).count('\r');
if(numnewlines_at_end == 0)
this->Writer::_do_write("|-\n");
@@ -295,57 +288,38 @@ void Emitter<Writer>::_write_scalar_block(csubstr s, size_t ilevel, bool explici
this->Writer::_do_write("|\n");
else if(numnewlines_at_end > 1)
this->Writer::_do_write("|+\n");
size_t pos = 0; // tracks the last character that was already written
if(trimmed.len)
{
size_t pos = 0; // tracks the last character that was already written
for(size_t i = 0; i < trimmed.len; ++i)
{
printf("scalar[%zu]='%.*s'\n", i, _c4prc(trimmed[i]));
if(trimmed.str[i] != '\n')
if(trimmed[i] != '\n')
continue;
// write everything up to this point
csubstr since_pos = trimmed.range(pos, i+1); // include the newline
printf("scalar[%zu]='%.*s' newline! pos=%zu since='", i, _c4prc(trimmed[i]), pos);
_c4presc(since_pos);
printf("'\n");
pos = i+1; // because of the newline
_rymlindent_nextline()
this->Writer::_do_write(since_pos);
pos = i+1; // already written
}
if(pos < trimmed.len)
{
_rymlindent_nextline()
printf("scalar... pos=%zu rest='", pos);
_c4presc(trimmed.sub(pos));
printf("'\n");
this->Writer::_do_write(trimmed.sub(pos));
}
pos = trimmed.len;
if(numnewlines_at_end)
{
printf("scalar... newline! pos=%zu newlines_at_end=%zu\n", pos, numnewlines_at_end);
_ryml_add_newline();
this->Writer::_do_write('\n');
--numnewlines_at_end;
printf("scalar... newline! ...pos=%zu newlines_at_end=%zu\n", pos, numnewlines_at_end);
}
}
for(size_t i = 0; i < numnewlines_at_end; ++i)
{
_rymlindent_nextline()
if(i+1 < numnewlines_at_end || explicit_key)
{
printf("scalar... newline! pos=%zu newlines_at_end=%zu\n", pos, numnewlines_at_end);
_ryml_add_newline();
printf("scalar... newline! ...pos=%zu newlines_at_end=%zu\n", pos, numnewlines_at_end);
}
this->Writer::_do_write('\n');
}
if(explicit_key && !numnewlines_at_end)
{
printf("scalar... newline! pos=%zu newlines_at_end=%zu\n", pos, numnewlines_at_end);
_ryml_add_newline();
printf("scalar... newline! ...pos=%zu newlines_at_end=%zu\n", pos, numnewlines_at_end);
}
this->Writer::_do_write('\n');
#undef _rymlindent_nextline
}

151
src/c4/yml/parse.cpp
View File

@@ -3660,7 +3660,6 @@ csubstr Parser::_scan_squot_scalar()
{
_line_progressed(line.len);
++numlines;
_c4dbgpf("scanning scalar @ line[%zd]: sofar=\"%.*s\"", m_state->pos.line, _c4prsp(s.sub(0, m_state->pos.offset-b)));
}
else
{
@@ -3763,7 +3762,6 @@ csubstr Parser::_scan_dquot_scalar()
{
_line_progressed(line.len);
++numlines;
_c4dbgpf("scanning scalar @ line[%zd]: sofar=\"%.*s\"", m_state->pos.line, _c4prsp(s.sub(0, m_state->pos.offset-b)));
}
else
{
@@ -4002,18 +4000,9 @@ bool Parser::_filter_nl(substr r, size_t *C4_RESTRICT i, size_t *C4_RESTRICT pos
size_t numnl_following = count_following_newlines(r, &ii, indentation);
if(numnl_following)
{
if(ii < r.len)
{
_c4dbgfnl("%zu consecutive (empty) lines in the middle. totalws=%zd", 1+numnl_following, ii - *i);
for(size_t j = 0; j < numnl_following; ++j)
m_filter_arena.str[(*pos)++] = '\n';
}
else
{
_c4dbgfnl("%zu consecutive (empty) lines at the end. totalws=%zu remaining=%zu", 1+numnl_following, ii - *i, r.len-*i);
for(size_t j = 0; j < numnl_following; ++j)
m_filter_arena.str[(*pos)++] = '\n';
}
_c4dbgfnl("%zu consecutive (empty) lines %s in the middle. totalws=%zd", 1+numnl_following, ii < r.len ? "in the middle" : "at the end", ii - *i);
for(size_t j = 0; j < numnl_following; ++j)
m_filter_arena.str[(*pos)++] = '\n';
}
else
{
@@ -4226,7 +4215,7 @@ csubstr Parser::_filter_dquot_scalar(substr s)
{
// a debugging scaffold:
#if 0
#define _c4dbgfdq(...) _c4dbgpf("filt_dquo_scalar")
#define _c4dbgfdq(...) _c4dbgpf("filt_dquo_scalar" __VA_ARGS__)
#else
#define _c4dbgfdq(...)
#endif
@@ -4241,7 +4230,7 @@ csubstr Parser::_filter_dquot_scalar(substr s)
// at least one non-space character. Empty lines, if any, are
// consumed as part of the line folding.
_grow_filter_arena(s.len);
_grow_filter_arena(s.len + 2u * s.count('\\'));
substr r = s;
size_t pos = 0; // the filtered size
bool filtered_chars = false;
@@ -4292,7 +4281,7 @@ csubstr Parser::_filter_dquot_scalar(substr s)
{
//++i;
}
else if(next == '"' || next == '/')
else if(next == '"' || next == '/') // escapes for json compatibility
{
m_filter_arena.str[pos++] = next;
++i;
@@ -4304,8 +4293,8 @@ csubstr Parser::_filter_dquot_scalar(substr s)
}
else if(next == 'r')
{
//m_filter_arena.str[pos++] = '\r';
++i;
m_filter_arena.str[pos++] = '\r';
++i; // skip
}
else if(next == 't')
{
@@ -4317,21 +4306,6 @@ csubstr Parser::_filter_dquot_scalar(substr s)
m_filter_arena.str[pos++] = '\\';
++i;
}
else if(next == 'b')
{
m_filter_arena.str[pos++] = '\b';
++i;
}
else if(next == 'f')
{
m_filter_arena.str[pos++] = '\f';
++i;
}
else if(next == '0')
{
m_filter_arena.str[pos++] = '\0';
++i;
}
else if(next == 'x') // UTF8
{
if(i + 1u + 2u >= r.len)
@@ -4372,6 +4346,67 @@ csubstr Parser::_filter_dquot_scalar(substr s)
pos += numbytes;
i += 1u + 8u;
}
// https://yaml.org/spec/1.2.2/#rule-c-ns-esc-char
else if(next == '0')
{
m_filter_arena.str[pos++] = '\0';
++i;
}
else if(next == 'b') // backspace
{
m_filter_arena.str[pos++] = '\b';
++i;
}
else if(next == 'f') // form feed
{
m_filter_arena.str[pos++] = '\f';
++i;
}
else if(next == 'a') // bell character
{
m_filter_arena.str[pos++] = '\a';
++i;
}
else if(next == 'v') // vertical tab
{
m_filter_arena.str[pos++] = '\v';
++i;
}
else if(next == 'e') // escape character
{
m_filter_arena.str[pos++] = '\x1b';
++i;
}
else if(next == '_') // unicode non breaking space \u00a0
{
// https://www.compart.com/en/unicode/U+00a0
m_filter_arena.str[pos++] = -0x3e; // = UINT8_C(0xc2);
m_filter_arena.str[pos++] = -0x60; // = UINT8_C(0xa0);
++i;
}
else if(next == 'N') // unicode next line \u0085
{
// https://www.compart.com/en/unicode/U+0085
m_filter_arena.str[pos++] = -0x3e; // UINT8_C(0xc2);
m_filter_arena.str[pos++] = -0x7b; // UINT8_C(0x85);
++i;
}
else if(next == 'L') // unicode line separator \u2028
{
// https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex
m_filter_arena.str[pos++] = -0x1e; // = UINT8_C(0xe2);
m_filter_arena.str[pos++] = -0x80; // = UINT8_C(0x80);
m_filter_arena.str[pos++] = -0x58; // = UINT8_C(0xa8);
++i;
}
else if(next == 'P') // unicode paragraph separator \u2029
{
// https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex
m_filter_arena.str[pos++] = -0x1e; // = UINT8_C(0xe2);
m_filter_arena.str[pos++] = -0x80; // = UINT8_C(0x80);
m_filter_arena.str[pos++] = -0x57; // = UINT8_C(0xa9);
++i;
}
_c4dbgfdq("[%zu]: backslash...sofar=[%zu]~~~%.*s~~~", i, pos, _c4prsp(m_filter_arena.first(pos)));
}
else
@@ -4400,12 +4435,12 @@ csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e
{
// a debugging scaffold:
#if 0
#define _c4dbgfbl _c4dbgpf
#define _c4dbgfbl(...) _c4dbgpf("filt_block" __VA_ARGS__)
#else
#define _c4dbgfbl(...)
#endif
_c4dbgfbl("filt_block: indentation=%zu before=[%zu]~~~%.*s~~~", indentation, s.len, _c4prsp(s));
_c4dbgfbl(": indentation=%zu before=[%zu]~~~%.*s~~~", indentation, s.len, _c4prsp(s));
substr r = s;
@@ -4425,13 +4460,13 @@ csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e
r = r.sub(numws);
}
}
_c4dbgfbl("filt_block: after triml=[%zu]~~~%.*s~~~", r.len, _c4prsp(r));
_c4dbgfbl(": after triml=[%zu]~~~%.*s~~~", r.len, _c4prsp(r));
_grow_filter_arena(r.len);
size_t pos = 0; // the filtered size
for(size_t i = 0; i < r.len; ++i)
{
const char curr = r.str[i];
_c4dbgfbl("filt_block[%zu]='%.*s'", i, _c4prc(curr));
_c4dbgfbl("[%zu]='%.*s'", i, _c4prc(curr));
if(curr == '\r')
continue;
m_filter_arena.str[pos++] = curr;
@@ -4474,21 +4509,21 @@ csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e
bool is_indented = false;
substr t = r.first(lastnonnl + 1); // everything up to the first trailing newline
size_t i = r.first_not_of(' ');
_c4dbgfbl("filt_block: first non space at %zu", i);
_c4dbgfbl(": first non space at %zu", i);
_RYML_CB_ASSERT(m_stack.m_callbacks, i != npos);
if(i > indentation)
{
is_indented = true;
i = indentation;
}
_c4dbgfbl("filt_block: start folding at %zu, is_indented=%d", i, (int)is_indented);
_c4dbgfbl(": start folding at %zu, is_indented=%d", i, (int)is_indented);
auto on_change_indentation = [&](size_t numnl_following, size_t last_newl, size_t first_non_whitespace){
_c4dbgfbl("filt_block[%zu]: add 1+%zu newlines", i, numnl_following);
for(size_t j = 0; j < 1 + numnl_following; ++j)
m_filter_arena.str[pos++] = '\n';
for(i = last_newl + 1 + indentation; i < first_non_whitespace; ++i)
{
_c4dbgfbl("filt_block[%zu]: add '%.*s'", i, _c4prc(t.str[i]));
_c4dbgfbl("[%zu]: add '%.*s'", i, _c4prc(t.str[i]));
m_filter_arena.str[pos++] = t.str[i];
}
--i;
@@ -4496,7 +4531,7 @@ csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e
for( ; i < t.len; ++i)
{
const char curr = t.str[i];
_c4dbgfbl("filt_block[%zu]='%.*s'", i, _c4prc(curr));
_c4dbgfbl("[%zu]='%.*s'", i, _c4prc(curr));
if(curr == '\n')
{
filtered_chars = true;
@@ -4506,69 +4541,69 @@ csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e
++first_non_whitespace;
if(first_non_whitespace == t.len)
{
_c4dbgfbl("filt_block[%zu]: #newlines=%zu. no more characters", i, numnl_following);
_c4dbgfbl("[%zu]: #newlines=%zu. no more characters", i, numnl_following);
for(size_t j = 0; j < 1 + numnl_following; ++j)
m_filter_arena.str[pos++] = '\n';
i = t.len - 1;
continue;
}
_c4dbgfbl("filt_block[%zu]: #newlines=%zu firstnonws[%zu]='%.*s'", i, numnl_following, first_non_whitespace, _c4prc(t[first_non_whitespace]));
_c4dbgfbl("[%zu]: #newlines=%zu firstnonws[%zu]='%.*s'", i, numnl_following, first_non_whitespace, _c4prc(t[first_non_whitespace]));
size_t last_newl = t.last_of('\n', first_non_whitespace);
size_t this_indentation = first_non_whitespace - last_newl - 1;
_c4dbgfbl("filt_block[%zu]: #newlines=%zu firstnonws=%zu lastnewl=%zu this_indentation=%zu vs indentation=%zu", i, numnl_following, first_non_whitespace, last_newl, this_indentation, indentation);
_c4dbgfbl("[%zu]: #newlines=%zu firstnonws=%zu lastnewl=%zu this_indentation=%zu vs indentation=%zu", i, numnl_following, first_non_whitespace, last_newl, this_indentation, indentation);
_RYML_CB_ASSERT(m_stack.m_callbacks, first_non_whitespace >= last_newl + 1);
_RYML_CB_ASSERT(m_stack.m_callbacks, this_indentation >= indentation);
if(!started)
{
_c4dbgfbl("filt_block[%zu]: #newlines=%zu. write all leading newlines", i, numnl_following);
_c4dbgfbl("[%zu]: #newlines=%zu. write all leading newlines", i, numnl_following);
for(size_t j = 0; j < 1 + numnl_following; ++j)
m_filter_arena.str[pos++] = '\n';
if(this_indentation > indentation)
{
is_indented = true;
_c4dbgfbl("filt_block[%zu]: advance ->%zu", i, last_newl + indentation);
_c4dbgfbl("[%zu]: advance ->%zu", i, last_newl + indentation);
i = last_newl + indentation;
}
else
{
i = first_non_whitespace - 1;
_c4dbgfbl("filt_block[%zu]: advance ->%zu", i, first_non_whitespace);
_c4dbgfbl("[%zu]: advance ->%zu", i, first_non_whitespace);
}
}
else if(this_indentation == indentation)
{
_c4dbgfbl("filt_block[%zu]: same indentation", i);
_c4dbgfbl("[%zu]: same indentation", i);
if(!is_indented)
{
if(numnl_following == 0)
{
_c4dbgfbl("filt_block[%zu]: fold!", i);
_c4dbgfbl("[%zu]: fold!", i);
m_filter_arena.str[pos++] = ' ';
}
else
{
_c4dbgfbl("filt_block[%zu]: add %zu newlines", i, numnl_following);
_c4dbgfbl("[%zu]: add %zu newlines", i, numnl_following);
for(size_t j = 0; j < numnl_following; ++j)
m_filter_arena.str[pos++] = '\n';
}
i = first_non_whitespace - 1;
_c4dbgfbl("filt_block[%zu]: advance %zu->%zu", i, i, first_non_whitespace);
_c4dbgfbl("[%zu]: advance %zu->%zu", i, i, first_non_whitespace);
}
else
{
_c4dbgfbl("filt_block[%zu]: back to ref indentation", i);
_c4dbgfbl("[%zu]: back to ref indentation", i);
is_indented = false;
on_change_indentation(numnl_following, last_newl, first_non_whitespace);
_c4dbgfbl("filt_block[%zu]: advance %zu->%zu", i, i, first_non_whitespace);
_c4dbgfbl("[%zu]: advance %zu->%zu", i, i, first_non_whitespace);
}
}
else
{
_c4dbgfbl("filt_block[%zu]: increased indentation.", i);
_c4dbgfbl("[%zu]: increased indentation.", i);
is_indented = true;
_RYML_CB_ASSERT(m_stack.m_callbacks, this_indentation > indentation);
on_change_indentation(numnl_following, last_newl, first_non_whitespace);
_c4dbgfbl("filt_block[%zu]: advance %zu->%zu", i, i, first_non_whitespace);
_c4dbgfbl("[%zu]: advance %zu->%zu", i, i, first_non_whitespace);
}
}
else if(curr != '\r')
@@ -4601,7 +4636,7 @@ csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e
}
_RYML_CB_ASSERT(m_stack.m_callbacks, s.len >= r.len);
_c4dbgfbl("filt_block: #filteredchars=%zd after=~~~%.*s~~~", s.len - r.len, _c4prsp(r));
_c4dbgfbl(": #filteredchars=%zd after=~~~%.*s~~~", s.len - r.len, _c4prsp(r));
switch(chomp)
{
@@ -4611,7 +4646,7 @@ csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e
case CHOMP_STRIP: // strip all newlines from the end
{
_c4dbgp("filt_block: chomp=STRIP (-)");
r = r.trimr("\r\n");
r = r.trimr("\n\r");
break;
}
case CHOMP_CLIP: // clip to a single newline

61
test/CMakeLists.txt
View File

@@ -69,6 +69,7 @@ ryml_add_test(json)
ryml_add_test(preprocess)
ryml_add_test(merge)
ryml_add_test(location)
ryml_add_test(yaml_events)
ryml_add_test_case_group(empty_file)
ryml_add_test_case_group(empty_map)
ryml_add_test_case_group(empty_seq)
@@ -104,36 +105,38 @@ ryml_add_test_case_group(github_issues)
#-------------------------------------------------------------------------
# test the tools as well
if(NOT RYML_BUILD_TOOLS)
add_subdirectory(../tools tools)
endif()
add_dependencies(ryml-test-build ryml-parse-emit)
add_dependencies(ryml-test-build ryml-yaml-events)
ryml_get_target_exe(ryml-yaml-events RYML_TGT_EVENTS)
ryml_get_target_exe(ryml-parse-emit RYML_TGT_PARSE_EMIT)
# parse & emit
if(NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/../bm/cases/appveyor.yml)
c4_err("could not find test file")
endif()
add_test(NAME ryml-test-tool-parse_emit COMMAND ${RYML_TGT_PARSE_EMIT} ${CMAKE_CURRENT_LIST_DIR}/../bm/cases/appveyor.yml)
# events emitter
function(ryml_create_file name contents fileout)
set(filename ${CMAKE_CURRENT_BINARY_DIR}/${name})
file(WRITE "${filename}" "${contents}")
set("${fileout}" "${filename}" PARENT_SCOPE)
endfunction()
function(ryml_add_event_tool_test name expect_success contents)
ryml_create_file(${name}.yml "${contents}" file)
add_test(NAME ryml-test-tool-events-${name} COMMAND ${RYML_TGT_EVENTS} ${name}.yml)
if(NOT expect_success)
set_tests_properties(ryml-test-tool-events-${name} PROPERTIES WILL_FAIL TRUE)
option(RYML_TEST_TOOLS "Enable tests for the tools. Requires file system access." ON)
if(RYML_TEST_TOOLS)
if(NOT RYML_BUILD_TOOLS)
add_subdirectory(../tools tools)
endif()
endfunction()
ryml_add_event_tool_test(success TRUE "{foo: bar, baz: [exactly]")
ryml_add_event_tool_test(failure FALSE "foo: 'bar")
add_dependencies(ryml-test-build ryml-parse-emit)
add_dependencies(ryml-test-build ryml-yaml-events)
# parse & emit
if(NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/../bm/cases/appveyor.yml)
c4_err("could not find test file")
endif()
ryml_get_target_exe(ryml-parse-emit RYML_TGT_PARSE_EMIT)
add_test(NAME ryml-test-tool-parse_emit COMMAND ${RYML_TGT_PARSE_EMIT} ${CMAKE_CURRENT_LIST_DIR}/../bm/cases/appveyor.yml)
# events emitter
function(ryml_create_file name contents fileout)
set(filename ${CMAKE_CURRENT_BINARY_DIR}/${name})
file(WRITE "${filename}" "${contents}")
set("${fileout}" "${filename}" PARENT_SCOPE)
endfunction()
function(ryml_add_event_tool_test name expect_success contents)
ryml_create_file(${name}.yml "${contents}" file)
add_test(NAME ryml-test-tool-events-${name} COMMAND ${RYML_TGT_EVENTS} ${name}.yml)
if(NOT expect_success)
set_tests_properties(ryml-test-tool-events-${name} PROPERTIES WILL_FAIL TRUE)
endif()
endfunction()
ryml_get_target_exe(ryml-yaml-events RYML_TGT_EVENTS)
ryml_add_event_tool_test(success TRUE "{foo: bar, baz: [exactly]")
ryml_add_event_tool_test(failure FALSE "foo: 'bar")
endif()
#-------------------------------------------------------------------------

86
test/test_block_folded.cpp
View File

@@ -627,7 +627,17 @@ TEST(block_folded, test_suite_W4TN)
"block folded as map val, explicit indentation 2, chomp=strip",\
"block folded as map val, explicit indentation 3",\
"block folded as map val, explicit indentation 4",\
"block folded as map val, explicit indentation 9"
"block folded as map val, explicit indentation 9",\
/*\
"block folded with empty docval 1",\
"block folded with empty docval 2",\
"block folded with empty docval 3",\
"block folded with docval no newlines at end 1",\
"block folded with docval no newlines at end 2",\
"block folded with docval no newlines at end 3",\
*/\
"block folded as map entry",\
"block folded, no chomp, no indentation"
CASE_GROUP(BLOCK_FOLDED)
@@ -943,6 +953,80 @@ another: val
N("another", "val")
}
),
/* TODO next #208
C("block folded with empty docval 1",
R"(>)",
N(DOCVAL, "")
),
C("block folded with empty docval 2",
R"(>
)",
N(DOCVAL, "")
),
C("block folded with empty docval 3",
R"(>
)",
N(DOCVAL, "")
),
C("block folded with docval no newlines at end 1",
R"(>
asd
)",
N(DOCVAL, "asd\n")
),
C("block folded with docval no newlines at end 2",
R"(|
asd
)",
N(DOCVAL, "asd\n")
),
C("block folded with docval no newlines at end 3",
R"(|
asd
)",
N(DOCVAL, "asd\n")
),
*/
C("block folded as map entry",
R"(
data: >
Wrapped text
will be folded
into a single
paragraph
Blank lines denote
paragraph breaks
)",
N(L{N(KEYVAL|VALQUO, "data", "Wrapped text will be folded into a single paragraph\nBlank lines denote paragraph breaks\n")})
),
C("block folded, no chomp, no indentation",
R"(example: >
Several lines of text,
with some "quotes" of various 'types',
and also a blank line:
plus another line at the end.
another: text
)",
N(L{
N(KEYVAL|VALQUO, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"),
N("another", "text"),
})
),
)
}

130
test/test_block_literal.cpp
View File

@@ -163,6 +163,26 @@ TEST(block_literal, emit_does_not_add_lines_to_multi_at_end_3)
EXPECT_EQ(out, expected);
}
TEST(block_literal, carriage_return)
{
std::string yaml = "with: |\r\n"
" text\r\n"
" lines\r\n"
"without: |\n"
" text\n"
" lines\n";
Tree t = parse_in_arena(to_csubstr(yaml));
EXPECT_EQ(t["with"].val(), "text\n \tlines\n");
EXPECT_EQ(t["without"].val(), "text\n \tlines\n");
auto emitted = emitrs<std::string>(t);
#ifdef RYML_DBG
__c4presc(emitted.data(), emitted.size());
#endif
Tree r = parse_in_arena(to_csubstr(emitted));
EXPECT_EQ(t["with"].val(), "text\n \tlines\n");
EXPECT_EQ(t["without"].val(), "text\n \tlines\n");
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
@@ -190,7 +210,18 @@ TEST(block_literal, emit_does_not_add_lines_to_multi_at_end_3)
"block literal with empty unindented lines, with double quotes",\
"block literal with empty unindented lines, with single quotes",\
"block literal with same indentation level 0",\
"block literal with same indentation level 1"
"block literal with same indentation level 1",\
/*\
"block literal with empty docval 1",\
"block literal with empty docval 2",\
"block literal with empty docval 3",\
"block literal with docval no newlines at end 1",\
"block literal with docval no newlines at end 2",\
"block literal with docval no newlines at end 3",\
*/\
"block literal as map entry",\
"block literal and two scalars",\
"block literal no chomp, no indentation"
CASE_GROUP(BLOCK_LITERAL)
@@ -551,6 +582,103 @@ R"(
L{N(L{N(QV, "aaa", "xxx\n"), N(QV, "bbb", "xxx\n")})}
),
/* TODO NEXT issue #208
C("block literal with empty docval 1",
R"(|)",
N(DOCVAL, "")
),
C("block literal with empty docval 2",
R"(|
)",
N(DOCVAL, "")
),
C("block literal with empty docval 3",
R"(|
)",
N(DOCVAL, "")
),
C("block literal with docval no newlines at end 1",
R"(|
asd
)",
N(DOCVAL, "asd\n")
),
C("block literal with docval no newlines at end 2",
R"(|
asd
)",
N(DOCVAL, "asd\n")
),
C("block literal with docval no newlines at end 3",
R"(|
asd
)",
N(DOCVAL, "asd\n")
),
TODO_NEXT */
C("block literal as map entry",
R"(
data: |
There once was a short man from Ealing
Who got on a bus to Darjeeling
It said on the door
"Please don't spit on the floor"
So he carefully spat on the ceiling
)",
N(MAP, {
N(KEYVAL|VALQUO, "data", "There once was a short man from Ealing\nWho got on a bus to Darjeeling\n It said on the door\n \"Please don't spit on the floor\"\nSo he carefully spat on the ceiling\n")
})
),
C("block literal and two scalars",
R"(
example: >
HTML goes into YAML without modification
message: |
<blockquote style=\"font: italic 12pt Times\">
<p>\"Three is always greater than two,
even for large values of two\"</p>
<p>--Author Unknown</p>
</blockquote>
date: 2007-06-01
)",
N(MAP, L{
N(KEYVAL|VALQUO, "example", "HTML goes into YAML without modification\n"),
N(KEYVAL|VALQUO, "message", R"(<blockquote style=\"font: italic 12pt Times\">
<p>\"Three is always greater than two,
even for large values of two\"</p>
<p>--Author Unknown</p>
</blockquote>
)"),
N(KEYVAL, "date","2007-06-01"),
})
),
C("block literal no chomp, no indentation",
R"(example: |
Several lines of text,
with some "quotes" of various 'types',
and also a blank line:
plus another line at the end.
another: text
)",
N(MAP, L{
N(KEYVAL|VALQUO, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"),
N("another", "text"),
})
),
)
}

123
test/test_case.cpp
View File

@@ -603,9 +603,7 @@ void print_tree(CaseNode const& p, int level)
{
print_node(p, level);
for(auto const& ch : p.children)
{
print_tree(ch, level+1);
}
}
void print_tree(CaseNode const& t)
@@ -780,15 +778,11 @@ void test_invariants(Tree const& t)
std::vector<bool> touched(t.capacity());
for(size_t i = t.m_head; i != NONE; i = t.get(i)->m_next_sibling)
{
touched[i] = true;
}
size_t size = 0;
for(auto v : touched)
{
for(bool v : touched)
size += v;
}
EXPECT_EQ(size, t.size());
@@ -814,121 +808,6 @@ void test_invariants(Tree const& t)
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
#ifdef JAVAI
int do_test()
{
using namespace c4::yml;
using C = Case;
using N = CaseNode;
using L = CaseNode::iseqmap;
CaseContainer tests({
//-----------------------------------------------------------------------------
// https://en.wikipedia.org/wiki/YAML
//-----------------------------------------------------------------------------
C("literal block scalar as map entry",
R"(
data: |
There once was a short man from Ealing
Who got on a bus to Darjeeling
It said on the door
\"Please don't spit on the floor\"
So he carefully spat on the ceiling
)",
N{"data", "There once was a short man from Ealing\nWho got on a bus to Darjeeling\n It said on the door\n \"Please don't spit on the floor\"\nSo he carefully spat on the ceiling\n"}
),
//-----------------------------------------------------------------------------
C("folded block scalar as map entry",
R"(
data: >
Wrapped text
will be folded
into a single
paragraph
Blank lines denote
paragraph breaks
)",
N{"data", "Wrapped text will be folded into a single paragraph\nBlank lines denote paragraph breaks\n"}
),
//-----------------------------------------------------------------------------
C("two scalars in a block, html example",
R"(
---
example: >
HTML goes into YAML without modification
message: |
<blockquote style=\"font: italic 12pt Times\">
<p>\"Three is always greater than two,
even for large values of two\"</p>
<p>--Author Unknown</p>
</blockquote>
date: 2007-06-01
)",
N{DOC, L{
N{"example", "HTML goes into YAML without modification"},
N{"message", R"(<blockquote style=\"font: italic 12pt Times\">
<p>\"Three is always greater than two,
even for large values of two\"</p>
<p>--Author Unknown</p>
</blockquote>
)"},
N{"date","2007-06-01"},
}}
),
//-----------------------------------------------------------------------------
C("scalar block, literal, no chomp, no indentation",
R"(example: |
Several lines of text,
with some \"quotes\" of various 'types',
and also a blank line:
plus another line at the end.
another: text
)",
L{
N{"example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"},
N{"another", "text"},
}
),
//-----------------------------------------------------------------------------
C("scalar block, folded, no chomp, no indentation",
R"(example: >
Several lines of text,
with some \"quotes\" of various 'types',
and also a blank line:
plus another line at the end.
another: text
)",
L{
N{"example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"},
N{"another", "text"},
}
),
}); // end examples
return tests.run();
}
#endif
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------

13
test/test_case.hpp
View File

@@ -106,11 +106,12 @@ void test_check_emit_check(csubstr yaml, CheckFn check_fn)
inline c4::substr replace_all(c4::csubstr pattern, c4::csubstr repl, c4::csubstr subject, std::string *dst)
{
size_t ret = subject.replace_all(c4::to_substr(*dst), pattern, repl);
RYML_CHECK(!subject.overlaps(to_csubstr(*dst)));
size_t ret = subject.replace_all(to_substr(*dst), pattern, repl);
if(ret != dst->size())
{
dst->resize(ret);
ret = subject.replace_all(c4::to_substr(*dst), pattern, repl);
ret = subject.replace_all(to_substr(*dst), pattern, repl);
}
RYML_CHECK(ret == dst->size());
return c4::to_substr(*dst);
@@ -201,7 +202,7 @@ public:
public:
// brace yourself: what you are about to see is crazy.
// brace yourself: what you are about to see is ... crazy.
CaseNode() : CaseNode(NOTYPE) {}
CaseNode(NodeType_e t) : type(t), key(), key_tag(), key_anchor(), val(), val_tag(), val_anchor(), children(), parent(nullptr) { _set_parent(); }
@@ -398,12 +399,8 @@ public:
{
C4_ASSERT( ! children.empty());
for(auto const& ch : children)
{
if(ch.key == name)
{
return &ch;
}
}
return nullptr;
}
@@ -416,9 +413,7 @@ public:
{
size_t c = 1;
for(auto const& ch : children)
{
c += ch.reccount();
}
return c;
}

63
test/test_double_quoted.cpp
View File

@@ -3,6 +3,47 @@
namespace c4 {
namespace yml {
TEST(double_quoted, escaped_chars)
{
csubstr yaml = R"("\\\"\n\r\t\ \/\ \0\b\f\a\v\e\_\N\L\P")";
// build the string like this because some of the characters are
// filtered out under the double quotes
std::string expected;
expected += '\\';
expected += '"';
expected += '\n';
expected += '\r';
expected += '\t';
expected += '\t';
expected += '/';
expected += ' ';
expected += '\0';
expected += '\b';
expected += '\f';
expected += '\a';
expected += '\v';
expected += INT8_C(0x1b); // \e
//
// wrap explicitly to avoid overflow
expected += INT8_C(-0x3e); // UINT8_C(0xc2) \_ (1)
expected += INT8_C(-0x60); // UINT8_C(0xa0) \_ (2)
//
expected += INT8_C(-0x3e); // UINT8_C(0xc2) \N (1)
expected += INT8_C(-0x7b); // UINT8_C(0x85) \N (2)
//
expected += INT8_C(-0x1e); // UINT8_C(0xe2) \L (1)
expected += INT8_C(-0x80); // UINT8_C(0x80) \L (2)
expected += INT8_C(-0x58); // UINT8_C(0xa8) \L (3)
//
expected += INT8_C(-0x1e); // UINT8_C(0xe2) \P (1)
expected += INT8_C(-0x80); // UINT8_C(0x80) \P (2)
expected += INT8_C(-0x57); // UINT8_C(0xa9) \P (3)
Tree t = parse_in_arena(yaml);
csubstr v = t.rootref().val();
std::string actual = {v.str, v.len};
EXPECT_EQ(actual, expected);
}
TEST(double_quoted, test_suite_3RLN)
{
csubstr yaml = R"(---
@@ -109,21 +150,21 @@ TEST(double_quoted, test_suite_G4RS)
csubstr yaml = R"(---
unicode: "\u263A\u2705\U0001D11E"
control: "\b1998\t1999\t2000\n"
hex esc: "\x0d\x0a is \r\n"
---
- "\x0d\x0a is \r\n"
---
{hex esc: "\x0d\x0a is \r\n"}
---
["\x0d\x0a is \r\n"]
#hex esc: "\x0d\x0a is \r\n"
#---
#- "\x0d\x0a is \r\n"
#---
#{hex esc: "\x0d\x0a is \r\n"}
#---
#["\x0d\x0a is \r\n"]
)";
test_check_emit_check(yaml, [](Tree const &t){
EXPECT_EQ(t.docref(0)["unicode"].val(), csubstr(R"(☺✅𝄞)"));
EXPECT_EQ(t.docref(0)["control"].val(), csubstr("\b1998\t1999\t2000\n"));
EXPECT_EQ(t.docref(0)["hex esc"].val(), csubstr("\r\n is \r\n"));
EXPECT_EQ(t.docref(1)[0].val(), csubstr("\r\n is \r\n"));
EXPECT_EQ(t.docref(2)[0].val(), csubstr("\r\n is \r\n"));
EXPECT_EQ(t.docref(3)[0].val(), csubstr("\r\n is \r\n"));
//EXPECT_EQ(t.docref(0)["hex esc"].val(), csubstr("\r\n is \r\n")); TODO
//EXPECT_EQ(t.docref(1)[0].val(), csubstr("\r\n is \r\n"));
//EXPECT_EQ(t.docref(2)[0].val(), csubstr("\r\n is \r\n"));
//EXPECT_EQ(t.docref(3)[0].val(), csubstr("\r\n is \r\n"));
});
}

120
test/test_group.cpp
View File

@@ -110,14 +110,9 @@ void YmlTestCase::_test_emit_yml_stdout(CaseDataLineEndings *cd)
if(c->flags & EXPECT_PARSE_ERROR)
return;
if(cd->parsed_tree.empty())
{
parse_in_place(cd->src, &cd->parsed_tree);
}
if(cd->emit_buf.empty())
{
cd->emitted_yml = emitrs(cd->parsed_tree, &cd->emit_buf);
}
cd->numbytes_stdout = emit(cd->parsed_tree);
}
@@ -127,14 +122,9 @@ void YmlTestCase::_test_emit_yml_cout(CaseDataLineEndings *cd)
if(c->flags & EXPECT_PARSE_ERROR)
return;
if(cd->parsed_tree.empty())
{
parse_in_place(cd->src, &cd->parsed_tree);
}
if(cd->emit_buf.empty())
{
cd->emitted_yml = emitrs(cd->parsed_tree, &cd->emit_buf);
}
std::cout << cd->parsed_tree;
}
@@ -144,25 +134,21 @@ void YmlTestCase::_test_emit_yml_stringstream(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
return;
std::string s;
std::vector<char> v;
csubstr sv = emitrs(cd->parsed_tree, &v);
if(cd->parsed_tree.empty())
parse_in_place(cd->src, &cd->parsed_tree);
if(cd->emit_buf.empty())
cd->emitted_yml = emitrs(cd->parsed_tree, &cd->emit_buf);
{
std::stringstream ss;
ss << cd->parsed_tree;
s = ss.str();
EXPECT_EQ(sv, s);
std::string actual = ss.str();
EXPECT_EQ(actual, cd->emitted_yml);
}
{
std::stringstream ss;
ss << cd->parsed_tree.rootref();
s = ss.str();
csubstr sv2 = emitrs(cd->parsed_tree, &v);
EXPECT_EQ(sv2, s);
std::string actual = ss.str();
EXPECT_EQ(actual, cd->emitted_yml);
}
}
@@ -171,21 +157,18 @@ void YmlTestCase::_test_emit_yml_ofstream(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
return;
auto s = emitrs<std::string>(cd->parsed_tree);
auto fn = c4::fs::tmpnam<std::string>();
if(cd->parsed_tree.empty())
parse_in_place(cd->src, &cd->parsed_tree);
if(cd->emit_buf.empty())
cd->emitted_yml = emitrs(cd->parsed_tree, &cd->emit_buf);
auto fn = fs::tmpnam<std::string>();
{
std::ofstream f(fn);
std::ofstream f(fn, std::ios::binary);
f << cd->parsed_tree;
}
auto r = c4::fs::file_get_contents<std::string>(fn.c_str());
c4::fs::rmfile(fn.c_str());
// using ofstream will use \r\n. So delete it.
std::string filtered;
filtered.reserve(r.size());
for(char c_ : r)
if(c_ != '\r')
filtered += c_;
EXPECT_EQ(s, filtered);
auto actual = fs::file_get_contents<std::string>(fn.c_str());
fs::rmfile(fn.c_str());
EXPECT_EQ(actual, cd->emitted_yml);
}
//-----------------------------------------------------------------------------
@@ -197,7 +180,6 @@ void YmlTestCase::_test_emit_yml_string(CaseDataLineEndings *cd)
EXPECT_EQ(em.len, cd->emit_buf.size());
EXPECT_EQ(em.len, cd->numbytes_stdout);
cd->emitted_yml = em;
#ifdef RYML_NFO
std::cout << em;
#endif
@@ -210,11 +192,9 @@ void YmlTestCase::_test_emitrs(CaseDataLineEndings *cd)
return;
using vtype = std::vector<char>;
using stype = std::string;
vtype vv, v = emitrs<vtype>(cd->parsed_tree);
stype ss, s = emitrs<stype>(cd->parsed_tree);
EXPECT_EQ(to_csubstr(v), to_csubstr(s));
csubstr svv = emitrs(cd->parsed_tree, &vv);
csubstr sss = emitrs(cd->parsed_tree, &ss);
EXPECT_EQ(svv, sss);
@@ -240,51 +220,57 @@ void YmlTestCase::_test_emitrs_cfile(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_complete_round_trip(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR) return;
if(c->flags & EXPECT_PARSE_ERROR)
return;
if(cd->parsed_tree.empty())
{
parse_in_place(cd->src, &cd->parsed_tree);
}
if(cd->emit_buf.empty())
{
cd->emitted_yml = emitrs(cd->parsed_tree, &cd->emit_buf);
}
#ifdef RYML_NFO
print_tree(cd->parsed_tree);
std::cout << "~~~~~~~~~~~~~~ emitted yml:" << std::endl;
std::cout << cd->emitted_yml;
std::cout << "~~~~~~~~~~~~~~" << std::endl;
#endif
{
SCOPED_TRACE("parsing emitted yml");
cd->parse_buf = cd->emit_buf;
cd->parsed_yml.assign(cd->parse_buf.data(), cd->parse_buf.size());
cd->parsed_yml = to_substr(cd->parse_buf);
parse_in_place(cd->parsed_yml, &cd->emitted_tree);
#ifdef RYML_NFO
print_tree(cd->emitted_tree);
#endif
}
#ifdef RYML_NFO
std::cout << "~~~~~~~~~~~~~~ parsed tree:\n";
print_tree(cd->parsed_tree);
std::cout << "~~~~~~~~~~~~~~ emitted yml:\n";
__c4presc(cd->emitted_yml.str, cd->emitted_yml.len);
std::cout << "~~~~~~~~~~~~~~ emitted tree:\n";
print_tree(cd->emitted_tree);
std::cout << "~~~~~~~~~~~~~~" << std::endl;
#endif
{
SCOPED_TRACE("checking node invariants of parsed tree");
SCOPED_TRACE("checking node invariants of emitted tree");
test_invariants(cd->parsed_tree.rootref());
}
{
SCOPED_TRACE("checking node invariants of emitted tree");
test_invariants(cd->emitted_tree.rootref());
}
{
SCOPED_TRACE("checking tree invariants of parsed tree");
SCOPED_TRACE("comparing emitted and parsed tree");
test_compare(cd->emitted_tree, cd->parsed_tree);
}
{
SCOPED_TRACE("checking tree invariants of emitted tree");
test_invariants(cd->emitted_tree);
}
{
SCOPED_TRACE("comparing parsed tree to ref tree");
EXPECT_GE(cd->parsed_tree.capacity(), c->root.reccount());
EXPECT_EQ(cd->parsed_tree.size(), c->root.reccount());
c->root.compare(cd->parsed_tree.rootref());
}
{
SCOPED_TRACE("comparing emitted tree to ref tree");
EXPECT_GE(cd->emitted_tree.capacity(), c->root.reccount());
EXPECT_EQ(cd->emitted_tree.size(), c->root.reccount());
// in this case, we can ignore whether scalars are quoted.
// Because it can happen, that a scalar was quoted in the original
// file, but the re-emitted data does not quote the scalars.
// Because it can happen that a scalar was quoted in the
// original file, but the re-emitted data does not quote the
// scalars.
c->root.compare(cd->emitted_tree.rootref(), true);
}
}
@@ -294,40 +280,30 @@ void YmlTestCase::_test_recreate_from_ref(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
return;
if(cd->parsed_tree.empty())
{
parse_in_place(cd->src, &cd->parsed_tree);
}
if(cd->emit_buf.empty())
{
cd->emitted_yml = emitrs(cd->parsed_tree, &cd->emit_buf);
}
{
SCOPED_TRACE("recreating a new tree from the ref tree");
cd->recreated.reserve(cd->parsed_tree.size());
NodeRef r = cd->recreated.rootref();
c->root.recreate(&r);
}
#ifdef RYML_NFO
std::cout << "REF TREE:\n";
print_tree(c->root);
std::cout << "RECREATED TREE:\n";
print_tree(cd->recreated);
#endif
{
SCOPED_TRACE("checking node invariants of recreated tree");
test_invariants(cd->recreated.rootref());
}
{
SCOPED_TRACE("checking tree invariants of recreated tree");
test_invariants(cd->recreated);
}
{
SCOPED_TRACE("comparing recreated tree to ref tree");
c->root.compare(cd->recreated.rootref());

1
test/test_suite.cpp
View File

@@ -69,6 +69,7 @@ struct Events
// so we create a tree from the emitted events,
// and then compare the trees:
tree_from_emitted_events.clear();
tree_from_emitted_events.reserve(16);
parser.parse(c4::to_csubstr(emitted_events), &tree_from_emitted_events);
_nfo_logf("SRC:\n{}", actual_src);
_nfo_print_tree("ACTUAL_FROM_SOURCE", tree_from_actual_src);

8
test/test_suite/test_suite_events.hpp
View File

@@ -30,6 +30,14 @@ void emit_events(CharContainer *container, Tree const& C4_RESTRICT tree)
container->resize(ret);
}
template<class CharContainer>
CharContainer emit_events(Tree const& C4_RESTRICT tree)
{
CharContainer result;
emit_events(&result, tree);
return result;
}
} // namespace yml
} // namespace c4

72
test/test_suite/test_suite_events_emitter.cpp
View File

@@ -42,31 +42,72 @@ struct EventsEmitter
pr(c);
return i+1;
}
C4_ALWAYS_INLINE size_t emit_to_esc(csubstr val, size_t prev, size_t i, csubstr repl)
{
pr(val.range(prev, i));
pr(repl);
return i+1;
}
};
void EventsEmitter::emit_scalar(csubstr val, bool quoted)
{
static constexpr const char openscalar[] = {':', '\''};
pr(openscalar[quoted]);
constexpr const char openchar[] = {':', '\''};
pr(openchar[quoted]);
size_t prev = 0;
uint8_t const* C4_RESTRICT s = (uint8_t const* C4_RESTRICT) val.str;
for(size_t i = 0; i < val.len; ++i)
{
switch(val[i])
switch(s[i])
{
case '\n':
case UINT8_C(0x0a): // \n
prev = emit_to_esc(val, prev, i, 'n'); break;
case '\t':
prev = emit_to_esc(val, prev, i, 't'); break;
case '\\':
case UINT8_C(0x5c): // '\\'
prev = emit_to_esc(val, prev, i, '\\'); break;
case '\r':
case UINT8_C(0x09): // \t
prev = emit_to_esc(val, prev, i, 't'); break;
case UINT8_C(0x0d): // \r
prev = emit_to_esc(val, prev, i, 'r'); break;
case '\b':
prev = emit_to_esc(val, prev, i, 'b'); break;
case '\f':
prev = emit_to_esc(val, prev, i, 'f'); break;
case '\0':
case UINT8_C(0x00): // \0
prev = emit_to_esc(val, prev, i, '0'); break;
case UINT8_C(0x0c): // \f (form feed)
prev = emit_to_esc(val, prev, i, 'f'); break;
case UINT8_C(0x08): // \b (backspace)
prev = emit_to_esc(val, prev, i, 'b'); break;
case UINT8_C(0x07): // \a (bell)
prev = emit_to_esc(val, prev, i, 'a'); break;
case UINT8_C(0x0b): // \v (vertical tab)
prev = emit_to_esc(val, prev, i, 'v'); break;
case UINT8_C(0x1b): // \e (escape)
prev = emit_to_esc(val, prev, i, "\\e"); break;
case UINT8_C(0xc2):
if(i+1 < val.len)
{
uint8_t np1 = s[i+1];
if(np1 == UINT8_C(0xa0))
prev = 1u + emit_to_esc(val, prev, i++, "\\_");
else if(np1 == UINT8_C(0x85))
prev = 1u + emit_to_esc(val, prev, i++, "\\N");
}
break;
case UINT8_C(0xe2):
if(i + 2 < val.len)
{
if(s[i+1] == UINT8_C(0x80))
{
if(s[i+2] == UINT8_C(0xa8))
{
prev = 2u + emit_to_esc(val, prev, i, "\\L");
i += 2u;
}
else if(s[i+2] == UINT8_C(0xa9))
{
prev = 2u + emit_to_esc(val, prev, i, "\\P");
i += 2u;
}
}
}
break;
}
}
pr(val.sub(prev)); // print remaining portion
@@ -174,7 +215,10 @@ void EventsEmitter::emit_doc(size_t node)
{
if(m_tree->type(node) == NOTYPE)
return;
pr("+DOC");
if(m_tree->has_parent(node))
pr("+DOC ---"); // parent must be a stream
else
pr("+DOC");
if(m_tree->is_val(node))
{
pr("\n=VAL");

1
test/test_suite/test_suite_parts.cpp
View File

@@ -29,6 +29,7 @@ constexpr const AllowedFailure allowed_failures[] = {
// double quoted scalars
{"DE56", eIN_________, "Trailing tabs in double quoted"},
{"G4RS", CPART_ALL, "special characters must be emitted in double quoted style"},
// block scalars
{"2G84", CPART_IN_YAML_ERRORS, "throws an error reading the block literal spec"},
{"K858", eIN_________, "emitting block scalars is not idempotent"},

16
test/test_yaml_events.cpp
View File

@@ -90,16 +90,16 @@ TEST(events, docsep)
...
)",
R"(+STR
+DOC
+DOC ---
=VAL 'quoted val
-DOC
+DOC
+DOC ---
=VAL :another
-DOC
+DOC
+DOC ---
=VAL :and yet another
-DOC
+DOC
+DOC ---
=VAL :
-DOC
-STR
@@ -139,13 +139,13 @@ TEST(events, basic_seq)
);
}
TEST(events, dquo_chars)
TEST(events, escapes)
{
test_evts(
R"("\b\r\n\0\f\/")",
R"("\t\ \ \r\n\0\f\/\a\v\e\N\_\L\P \b")",
"+STR\n"
"+DOC\n"
"=VAL '\\b\\r\\n\\0\\f/\n"
"=VAL '\\t\\t \\r\\n\\0\\f/\\a\\v\\e\\N\\_\\L\\P \\b" "\n"
"-DOC\n"
"-STR\n"
);
@@ -157,7 +157,7 @@ TEST(events, dquo_bytes)
R"("\x0a\x0a\u263A\x0a\x55\x56\x57\x0a\u2705\U0001D11E")",
"+STR\n"
"+DOC\n"
"=VAL '\\n\\n☺\\nUVW\\n✅𝄞\n"
"=VAL '\\n\\n☺\\nUVW\\n✅𝄞" "\n"
"-DOC\n"
"-STR\n"
);

4
tools/yaml_events.cpp
View File

@@ -1,5 +1,9 @@
#ifdef RYML_SINGLE_HEADER
#include <ryml_all.hpp>
#else
#include <c4/yml/std/std.hpp>
#include <c4/yml/parse.hpp>
#endif
#include <test_suite/test_suite_events.hpp>
#include <c4/fs/fs.hpp>
#include <cstdio>