Mirrors/rapidyaml
0
0
Fork 0
mirror of https://github.com/biojppm/rapidyaml.git synced 2026-01-18 13:31:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

[impl] add tag lookup

Browse Source
This commit is contained in:
Joao Paulo Magalhaes
2022-02-11 11:18:26 +00:00
parent 185615c1c1
commit d4e45195fd
18 changed files with 1042 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CONTRIBUTING.md
View File

@@ -8,9 +8,6 @@ Thanks for your contribution!
when calling cmake. To enable only tests, use `-DRYML_BUILD_TESTS=ON`; to
enable only benchmarks use `-DRYML_BUILD_BENCHMARKS=ON`. All these flags
are disabled by default.
* Submitted pull requests should target the `dev` branch. (The `master`
branch is the stable branch, and merges from `dev` to `master` are done
only if the tests succeed.)
* Code style for pull requests should respect the existing code style:
```c++
if(foo) // no space before parens

14
README.md
View File

@@ -919,20 +919,20 @@ See also [the roadmap](./ROADMAP.md) for a list of future work.
ryml deliberately makes no effort to follow the standard in the following situations:
* Containers are not accepted as mapping keys: keys must be scalars.
* Tab characters after `:` and `-` are not accepted tokens, unless
ryml is compiled with the macro `RYML_WITH_TAB_TOKENS`. This
requirement exists because checking for tabs introduces branching
into the parser's hot code and in some cases costs as much as 10%
in parsing time.
* Containers are not accepted as mapping keys: keys must be scalar strings.
* Tags are parsed as-is; tag lookup is not supported.
* Anchor names must not end with a terminating colon: eg `&anchor: key: val`.
* `%TAG` directives have no effect and are ignored. All schemas are assumed
to be the default YAML 2002 schema.
* `%YAML` directives have no effect and are ignored.
Some of the limitations above will be worked on, (eg tag
lookups). Others (notably container keys) most likely will not.
* `%TAG` directives are limited to a default maximum of 4 instances
per `Tree`. To increase this maximum, define the preprocessor symbol
`RYML_MAX_TAG_DIRECTIVES` to a suitable value. This arbitrary limit
reflects the usual practice of having at most 1 or 2 tag directives;
also, be aware that this feature is under consideration for removal
in YAML 1.3.
Also, ryml tends to be on the permissive side where the YAML standard
dictates there should be an error; in many of these cases, ryml will

2
changelog/current.md
View File

@@ -98,6 +98,8 @@ As part of the [new feature to track source locations](https://github.com/biojpp
assert(from_tag_long(TAG_MAP) == "<tag:yaml.org,2002:map>");
assert(normalize_tag_long("!!map") == "<tag:yaml.org,2002:map>");
```
- Add an experimental API to resolve tags based on the tree's tag directives. This API is still imature and will likely be subject to changes, so we won't document it yet.
- Regarding emit styles (see issue [#37](https://github.com/biojppm/rapidyaml/issues/37)): add an experimental API to force flow/block style on container nodes, as well as block-literal/block-folded/double-quoted/single-quoted/plain styles on scalar nodes. This API is also immature and will likely be subject to changes, so we won't document it yet. But if you are desperate for this functionality, the new facilities will let you go further.
### Fixes

27
src/c4/yml/emit.def.hpp
View File

@@ -78,10 +78,37 @@ void Emitter<Writer>::_emit_yaml(size_t id)
}
}
auto *btd = m_tree->tag_directives().b;
auto *etd = m_tree->tag_directives().e;
auto write_tag_directives = [&btd, etd, this](size_t next_node){
auto end = btd;
while(end < etd)
{
if(end->next_node_id > next_node)
break;
++end;
}
for( ; btd != end; ++btd)
{
if(next_node != m_tree->first_child(m_tree->parent(next_node)))
this->Writer::_do_write("...\n");
this->Writer::_do_write("%TAG ");
this->Writer::_do_write(btd->handle);
this->Writer::_do_write(' ');
this->Writer::_do_write(btd->prefix);
this->Writer::_do_write('\n');
}
};
if(m_tree->is_stream(id))
{
if(m_tree->first_child(id) != NONE)
write_tag_directives(m_tree->first_child(id));
for(size_t child = m_tree->first_child(id); child != NONE; child = m_tree->next_sibling(child))
{
dispatch(child);
if(m_tree->next_sibling(child) != NONE)
write_tag_directives(m_tree->next_sibling(child));
}
}
else if(m_tree->is_container(id))
{

46
src/c4/yml/parse.cpp
View File

@@ -1823,6 +1823,7 @@ bool Parser::_handle_map_blck()
return false;
}
//-----------------------------------------------------------------------------
bool Parser::_handle_top()
{
@@ -1840,7 +1841,7 @@ bool Parser::_handle_top()
if(trimmed.begins_with('%'))
{
_c4dbgpf("%% directive! ignoring...: '{}'", rem);
_handle_directive(trimmed);
_line_progressed(rem.len);
return true;
}
@@ -2192,6 +2193,7 @@ bool Parser::_handle_types()
csubstr scalar = _slurp_doc_scalar();
_c4dbgpf("docval. after slurp: {}, at node {}: '{}'", m_state->pos.offset, m_state->node_id, scalar);
m_tree->to_val(m_state->node_id, scalar, DOC);
_c4dbgpf("docval. val tag {} -> {}", m_val_tag, normalize_tag(m_val_tag));
m_tree->set_val_tag(m_state->node_id, normalize_tag(m_val_tag));
m_val_tag.clear();
if(!m_val_anchor.empty())
@@ -3147,9 +3149,10 @@ void Parser::_end_stream()
}
else if(m_tree->is_doc(m_state->node_id) || m_tree->type(m_state->node_id) == NOTYPE)
{
_c4dbgp("to docval...");
NodeType_e quoted = has_any(QSCL) ? VALQUO : NOTYPE; // do this before consuming the scalar
m_tree->to_val(m_state->node_id, _consume_scalar(), DOC|quoted);
csubstr scalar = _consume_scalar();
_c4dbgpf("node[{}]: to docval '{}'{}", m_state->node_id, scalar, quoted == VALQUO ? ", quoted" : "");
m_tree->to_val(m_state->node_id, scalar, DOC|quoted);
added = m_tree->get(m_state->node_id);
}
else
@@ -4978,6 +4981,43 @@ size_t Parser::_count_nlines(csubstr src)
return 1 + src.count('\n');
}
//-----------------------------------------------------------------------------
void Parser::_handle_directive(csubstr directive_)
{
csubstr directive = directive_;
if(directive.begins_with("%TAG"))
{
TagDirective td;
_c4dbgpf("%TAG directive: {}", directive_);
directive = directive.sub(4);
if(!directive.begins_with(' '))
_c4err("malformed tag directive: {}", directive_);
directive = directive.triml(' ');
size_t pos = directive.find(' ');
if(pos == npos)
_c4err("malformed tag directive: {}", directive_);
td.handle = directive.first(pos);
directive = directive.sub(td.handle.len).triml(' ');
pos = directive.find(' ');
if(pos != npos)
directive = directive.first(pos);
td.prefix = directive;
td.next_node_id = m_tree->size();
if(m_tree->size() > 0)
{
size_t prev = m_tree->size() - 1;
if(m_tree->is_root(prev) && m_tree->type(prev) != NOTYPE && !m_tree->is_stream(prev))
++td.next_node_id;
}
_c4dbgpf("%TAG: handle={} prefix={} next_node={}", td.handle, td.prefix, td.next_node_id);
m_tree->add_tag_directive(td);
}
else if(directive.begins_with("%YAML"))
{
_c4dbgpf("%YAML directive! ignoring...: {}", directive);
}
}
//-----------------------------------------------------------------------------
void Parser::set_flags(flag_t f, State * s)
{

2
src/c4/yml/parse.hpp
View File

@@ -363,6 +363,8 @@ private:
void _write_key_anchor(size_t node_id);
void _write_val_anchor(size_t node_id);
void _handle_directive(csubstr directive);
void _skipchars(char c);
template<size_t N>
void _skipchars(const char (&chars)[N]);

189
src/c4/yml/tree.cpp
View File

@@ -19,12 +19,7 @@ csubstr normalize_tag(csubstr tag)
if(tag.begins_with("!<"))
tag = tag.sub(1);
if(tag.begins_with("<!"))
{
size_t pos = tag.find('>');
if(pos == csubstr::npos)
return tag;
return tag.range(1, pos);
}
return tag;
return tag;
}
@@ -36,12 +31,7 @@ csubstr normalize_tag_long(csubstr tag)
if(tag.begins_with("!<"))
tag = tag.sub(1);
if(tag.begins_with("<!"))
{
size_t pos = tag.find('>');
if(pos == csubstr::npos)
return tag;
return tag.range(1, pos);
}
return tag;
return tag;
}
@@ -376,6 +366,8 @@ void Tree::_clear()
m_free_tail = 0;
m_arena = {};
m_arena_pos = 0;
for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i)
m_tag_directives[i] = {};
}
void Tree::_copy(Tree const& that)
@@ -400,6 +392,8 @@ void Tree::_copy(Tree const& that)
_relocate(arena); // does a memcpy of the arena and updates nodes using the old arena
m_arena = arena;
}
for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i)
m_tag_directives[i] = that.m_tag_directives[i];
}
void Tree::_move(Tree & that)
@@ -414,6 +408,8 @@ void Tree::_move(Tree & that)
m_free_tail = that.m_free_tail;
m_arena = that.m_arena;
m_arena_pos = that.m_arena_pos;
for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i)
m_tag_directives[i] = that.m_tag_directives[i];
that._clear();
}
@@ -437,6 +433,13 @@ void Tree::_relocate(substr next_arena)
if(in_arena(n->m_val.anchor))
n->m_val.anchor = _relocated(n->m_val.anchor, next_arena);
}
for(TagDirective &C4_RESTRICT td : m_tag_directives)
{
if(in_arena(td.prefix))
td.prefix = _relocated(td.prefix, next_arena);
if(in_arena(td.handle))
td.handle = _relocated(td.handle, next_arena);
}
}
@@ -495,6 +498,8 @@ void Tree::clear()
m_free_head = NONE;
m_free_tail = NONE;
}
for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i)
m_tag_directives[i] = {};
}
void Tree::_claim_root()
@@ -1675,6 +1680,166 @@ void Tree::to_stream(size_t node, type_bits more_flags)
}
//-----------------------------------------------------------------------------
size_t Tree::num_tag_directives() const
{
// this assumes we have a very small number of tag directives
for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i)
if(m_tag_directives[i].handle.empty())
return i;
return RYML_MAX_TAG_DIRECTIVES;
}
void Tree::clear_tag_directives()
{
for(TagDirective &td : m_tag_directives)
td = {};
}
size_t Tree::add_tag_directive(TagDirective const& td)
{
_RYML_CB_CHECK(m_callbacks, !td.handle.empty());
_RYML_CB_CHECK(m_callbacks, !td.prefix.empty());
_RYML_CB_ASSERT(m_callbacks, td.handle.begins_with('!'));
_RYML_CB_ASSERT(m_callbacks, td.handle.ends_with('!'));
// https://yaml.org/spec/1.2.2/#rule-ns-word-char
_RYML_CB_ASSERT(m_callbacks, td.handle == '!' || td.handle == "!!" || td.handle.trim('!').first_not_of("01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-") == npos);
size_t pos = num_tag_directives();
_RYML_CB_CHECK(m_callbacks, pos < RYML_MAX_TAG_DIRECTIVES);
m_tag_directives[pos] = td;
return pos;
}
size_t Tree::resolve_tag(substr output, csubstr tag, size_t node_id) const
{
// lookup from the end. We want to find the first directive that
// matches the tag and has a target node id leq than the given
// node_id.
for(size_t i = RYML_MAX_TAG_DIRECTIVES-1; i != (size_t)-1; --i)
{
auto const& td = m_tag_directives[i];
if(td.handle.empty())
continue;
if(tag.begins_with(td.handle) && td.next_node_id <= node_id)
{
_RYML_CB_ASSERT(m_callbacks, tag.len >= td.handle.len);
csubstr rest = tag.sub(td.handle.len);
size_t len = 1u + td.prefix.len + rest.len + 1u;
size_t numpc = rest.count('%');
if(numpc == 0)
{
if(len <= output.len)
{
output.str[0] = '<';
memcpy(1u + output.str, td.prefix.str, td.prefix.len);
memcpy(1u + output.str + td.prefix.len, rest.str, rest.len);
output.str[1u + td.prefix.len + rest.len] = '>';
}
}
else
{
// need to decode URI % sequences
size_t pos = rest.find('%');
_RYML_CB_ASSERT(m_callbacks, pos != npos);
do {
size_t next = rest.first_not_of("0123456789abcdefABCDEF", pos+1);
if(next == npos)
next = rest.len;
_RYML_CB_CHECK(m_callbacks, pos+1 < next);
_RYML_CB_CHECK(m_callbacks, pos+1 + 2 <= next);
size_t delta = next - (pos+1);
len -= delta;
pos = rest.find('%', pos+1);
} while(pos != npos);
if(len <= output.len)
{
size_t prev = 0, wpos = 0;
auto appendstr = [&](csubstr s) { memcpy(output.str + wpos, s.str, s.len); wpos += s.len; };
auto appendchar = [&](char c) { output.str[wpos++] = c; };
appendchar('<');
appendstr(td.prefix);
pos = rest.find('%');
_RYML_CB_ASSERT(m_callbacks, pos != npos);
do {
size_t next = rest.first_not_of("0123456789abcdefABCDEF", pos+1);
if(next == npos)
next = rest.len;
_RYML_CB_CHECK(m_callbacks, pos+1 < next);
_RYML_CB_CHECK(m_callbacks, pos+1 + 2 <= next);
uint8_t val;
if(C4_UNLIKELY(!read_hex(rest.range(pos+1, next), &val) || val > 127))
_RYML_CB_ERR(m_callbacks, "invalid URI character");
appendstr(rest.range(prev, pos));
appendchar((char)val);
prev = next;
pos = rest.find('%', pos+1);
} while(pos != npos);
_RYML_CB_ASSERT(m_callbacks, pos == npos);
_RYML_CB_ASSERT(m_callbacks, prev > 0);
_RYML_CB_ASSERT(m_callbacks, rest.len >= prev);
appendstr(rest.sub(prev));
appendchar('>');
_RYML_CB_ASSERT(m_callbacks, wpos == len);
}
}
return len;
}
}
return 0; // return 0 to signal that the tag is local and cannot be resolved
}
namespace {
csubstr _transform_tag(Tree *t, csubstr tag, size_t node)
{
size_t required_size = t->resolve_tag(substr{}, tag, node);
if(!required_size)
return tag;
const char *prev_arena = t->arena().str;
substr buf = t->alloc_arena(required_size);
_RYML_CB_ASSERT(t->m_callbacks, t->arena().str == prev_arena);
size_t actual_size = t->resolve_tag(buf, tag, node);
_RYML_CB_ASSERT(t->m_callbacks, actual_size <= required_size);
return buf.first(actual_size);
}
void _resolve_tags(Tree *t, size_t node)
{
for(size_t child = t->first_child(node); child != NONE; child = t->next_sibling(child))
{
if(t->has_key(child) && t->has_key_tag(child))
t->set_key_tag(child, _transform_tag(t, t->key_tag(child), child));
if(t->has_val(child) && t->has_val_tag(child))
t->set_val_tag(child, _transform_tag(t, t->val_tag(child), child));
_resolve_tags(t, child);
}
}
size_t _count_resolved_tags_size(Tree const* t, size_t node)
{
size_t sz = 0;
for(size_t child = t->first_child(node); child != NONE; child = t->next_sibling(child))
{
if(t->has_key(child) && t->has_key_tag(child))
sz += t->resolve_tag(substr{}, t->key_tag(child), child);
if(t->has_val(child) && t->has_val_tag(child))
sz += t->resolve_tag(substr{}, t->val_tag(child), child);
sz += _count_resolved_tags_size(t, child);
}
return sz;
}
} // namespace
void Tree::resolve_tags()
{
if(empty())
return;
if(num_tag_directives() == 0)
return;
size_t needed_size = _count_resolved_tags_size(this, root_id());
if(needed_size)
reserve_arena(arena_pos() + needed_size);
_resolve_tags(this, root_id());
}
//-----------------------------------------------------------------------------
csubstr Tree::lookup_result::resolved() const

55
src/c4/yml/tree.hpp
View File

@@ -108,13 +108,27 @@ typedef enum : tag_bits {
TAG_YAML = 15, /**< !!yaml Specify the default value of a mapping https://yaml.org/type/yaml.html */
} YamlTag_e;
YamlTag_e to_tag(csubstr tag);
csubstr from_tag(YamlTag_e tag);
csubstr from_tag_long(YamlTag_e tag);
csubstr normalize_tag(csubstr tag);
csubstr normalize_tag_long(csubstr tag);
struct TagDirective
{
/** Eg `!e!` in `%TAG !e! tag:example.com,2000:app/` */
csubstr handle;
/** Eg `tag:example.com,2000:app/` in `%TAG !e! tag:example.com,2000:app/` */
csubstr prefix;
/** The next node to which this tag directive applies */
size_t next_node_id;
};
#ifndef RYML_MAX_TAG_DIRECTIVES
/** the maximum number of tag directives in a Tree */
#define RYML_MAX_TAG_DIRECTIVES 4
#endif
//-----------------------------------------------------------------------------
@@ -152,6 +166,7 @@ typedef enum : type_bits {
DOCMAP = DOC|MAP,
DOCSEQ = DOC|SEQ,
DOCVAL = DOC|VAL,
// these flags are from a work in progress and should not be used yet
_WIP_STYLE_FLOW_SL = c4bit(14), ///< mark container with single-line flow format (seqs as '[val1,val2], maps as '{key: val, key2: val2}')
_WIP_STYLE_FLOW_ML = c4bit(15), ///< mark container with multi-line flow format (seqs as '[val1,\nval2], maps as '{key: val,\nkey2: val2}')
_WIP_STYLE_BLOCK = c4bit(16), ///< mark container with block format (seqs as '- val\n', maps as 'key: val')
@@ -248,6 +263,7 @@ public:
C4_ALWAYS_INLINE bool is_val_quoted() const { return (type & (VAL|VALQUO)) == (VAL|VALQUO); }
C4_ALWAYS_INLINE bool is_quoted() const { return (type & (KEY|KEYQUO)) == (KEY|KEYQUO) || (type & (VAL|VALQUO)) == (VAL|VALQUO); }
// these predicates are a work in progress and subject to change. Don't use yet.
C4_ALWAYS_INLINE bool default_block() const { return (type & (_WIP_STYLE_BLOCK|_WIP_STYLE_FLOW_ML|_WIP_STYLE_FLOW_SL)) == 0; }
C4_ALWAYS_INLINE bool marked_block() const { return (type & (_WIP_STYLE_BLOCK)) != 0; }
C4_ALWAYS_INLINE bool marked_flow_sl() const { return (type & (_WIP_STYLE_FLOW_SL)) != 0; }
@@ -730,6 +746,39 @@ public:
/** @} */
public:
/** @name tag directives */
/** @{ */
void resolve_tags();
size_t num_tag_directives() const;
size_t add_tag_directive(TagDirective const& td);
void clear_tag_directives();
size_t resolve_tag(substr output, csubstr tag, size_t node_id) const;
csubstr resolve_tag_sub(substr output, csubstr tag, size_t node_id) const
{
size_t needed = resolve_tag(output, tag, node_id);
return needed <= output.len ? output.first(needed) : output;
}
using tag_directive_const_iterator = TagDirective const*;
tag_directive_const_iterator begin_tag_directives() const { return m_tag_directives; }
tag_directive_const_iterator end_tag_directives() const { return m_tag_directives + num_tag_directives(); }
struct TagDirectiveProxy
{
tag_directive_const_iterator b, e;
tag_directive_const_iterator begin() const { return b; }
tag_directive_const_iterator end() const { return e; }
};
TagDirectiveProxy tag_directives() const { return TagDirectiveProxy{begin_tag_directives(), end_tag_directives()}; }
/** @} */
public:
/** @name modifying hierarchy */
@@ -966,7 +1015,7 @@ public:
* existing arena, and thus change the contents of individual nodes. */
substr alloc_arena(size_t sz)
{
if(sz >= arena_slack())
if(sz > arena_slack())
_grow_arena(sz - arena_slack());
substr s = _request_span(sz);
return s;
@@ -1325,6 +1374,8 @@ public:
Callbacks m_callbacks;
TagDirective m_tag_directives[RYML_MAX_TAG_DIRECTIVES];
};
} // namespace yml

23
test/test_case.cpp
View File

@@ -80,7 +80,14 @@ void test_compare(Tree const& actual, size_t node_actual,
EXPECT_EQ(actual.has_val_tag(node_actual), expected.has_val_tag(node_expected)) << _MORE_INFO;
if(actual.has_val_tag(node_actual) && expected.has_val_tag(node_expected))
{
EXPECT_EQ(actual.val_tag(node_actual), expected.val_tag(node_expected)) << _MORE_INFO;
auto filtered = [](csubstr tag) {
if(tag.begins_with("!<!") && tag.ends_with('>'))
return tag.offs(3, 1);
return tag;
};
csubstr actual_tag = filtered(actual.val_tag(node_actual));
csubstr expected_tag = filtered(actual.val_tag(node_expected));
EXPECT_EQ(actual_tag, expected_tag) << _MORE_INFO;
}
EXPECT_EQ(actual.has_key_anchor(node_actual), expected.has_key_anchor(node_expected)) << _MORE_INFO;
@@ -108,7 +115,7 @@ void test_compare(Tree const& actual, size_t node_actual,
void test_arena_not_shared(Tree const& a, Tree const& b)
{
for(NodeData *n = a.m_buf, *e = a.m_buf + a.m_cap; n != e; ++n)
for(NodeData const* n = a.m_buf, *e = a.m_buf + a.m_cap; n != e; ++n)
{
EXPECT_FALSE(b.in_arena(n->m_key.scalar)) << n - a.m_buf;
EXPECT_FALSE(b.in_arena(n->m_key.tag )) << n - a.m_buf;
@@ -117,7 +124,7 @@ void test_arena_not_shared(Tree const& a, Tree const& b)
EXPECT_FALSE(b.in_arena(n->m_val.tag )) << n - a.m_buf;
EXPECT_FALSE(b.in_arena(n->m_val.anchor)) << n - a.m_buf;
}
for(NodeData *n = b.m_buf, *e = b.m_buf + b.m_cap; n != e; ++n)
for(NodeData const* n = b.m_buf, *e = b.m_buf + b.m_cap; n != e; ++n)
{
EXPECT_FALSE(a.in_arena(n->m_key.scalar)) << n - b.m_buf;
EXPECT_FALSE(a.in_arena(n->m_key.tag )) << n - b.m_buf;
@@ -126,6 +133,16 @@ void test_arena_not_shared(Tree const& a, Tree const& b)
EXPECT_FALSE(a.in_arena(n->m_val.tag )) << n - b.m_buf;
EXPECT_FALSE(a.in_arena(n->m_val.anchor)) << n - b.m_buf;
}
for(TagDirective const& td : a.m_tag_directives)
{
EXPECT_FALSE(b.in_arena(td.handle));
EXPECT_FALSE(b.in_arena(td.prefix));
}
for(TagDirective const& td : b.m_tag_directives)
{
EXPECT_FALSE(a.in_arena(td.handle));
EXPECT_FALSE(a.in_arena(td.prefix));
}
}

106
test/test_suite.cpp
View File

@@ -115,6 +115,8 @@ struct Events
/** a processing level */
struct ProcLevel
{
size_t level;
ProcLevel *prev;
csubstr filename;
std::string src;
c4::yml::Parser parser;
@@ -126,8 +128,10 @@ struct ProcLevel
bool was_parsed = false;
bool was_emitted = false;
void init(csubstr filename_, csubstr src_, bool immutable_, bool reuse_)
void init(size_t level_, ProcLevel *prev_, csubstr filename_, csubstr src_, bool immutable_, bool reuse_)
{
level = level_;
prev = prev_;
filename = filename_;
src.assign(src_.begin(), src_.end());
immutable = immutable_;
@@ -136,15 +140,19 @@ struct ProcLevel
was_emitted = false;
}
void receive_src(ProcLevel & prev)
void receive_src(ProcLevel & prev_)
{
if(!prev.was_emitted)
prev.emit();
if(src != prev.emitted)
RYML_ASSERT(&prev_ == prev);
if(!prev_.was_emitted)
{
_nfo_logf("level[{}] not emitted. emit!", prev_.level);
prev_.emit();
}
if(src != prev_.emitted)
{
was_parsed = false;
was_emitted = false;
src = prev.emitted;
src = prev_.emitted;
}
}
@@ -163,7 +171,11 @@ struct ProcLevel
{
if(was_parsed)
return;
log("parsing source", src);
if(prev)
{
receive_src(*prev);
}
_nfo_logf("level[{}]: parsing source:\n{}", level, src);
if(reuse)
{
tree.clear();
@@ -179,10 +191,10 @@ struct ProcLevel
else
tree = parse_in_place(filename, c4::to_substr(src));
}
_nfo_print_tree("PARSED", tree);
tree.resolve_tags();
_nfo_print_tree("RESOLVED TAGS", tree);
was_parsed = true;
#if RYML_NFO
c4::yml::print_tree(tree);
#endif
//_resolve_if_needed();
}
@@ -195,9 +207,7 @@ struct ProcLevel
if(has_anchors_or_refs)
{
tree.resolve();
#if RYML_NFO
c4::yml::print_tree(tree);
#endif
_nfo_print_tree("RESOLVED", tree);
}
}
@@ -206,38 +216,60 @@ struct ProcLevel
if(was_emitted)
return;
if(!was_parsed)
{
_nfo_logf("level[{}] not parsed. parse!", level);
parse();
}
emitrs(tree, &emitted);
auto ss = c4::to_csubstr(emitted);
csubstr ss = to_csubstr(emitted);
if(ss.ends_with("\n...\n"))
emitted.resize(emitted.size() - 4);
log("emitted YAML", emitted);
was_emitted = true;
#if RYML_NFO
c4::log("EMITTED:\n{}", emitted);
#endif
_nfo_logf("EMITTED:\n{}", emitted);
}
void compare_trees(ProcLevel const& prev)
void compare_trees(ProcLevel & prev_)
{
RYML_ASSERT(&prev_ == prev);
if(!prev_.was_parsed)
{
_nfo_logf("level[{}] not parsed. parse!", prev_.level);
prev_.parse();
}
if(!was_parsed)
{
_nfo_logf("level[{}] not parsed. parse!", level);
parse();
#if RYML_NFO
c4::print("PREV:"); print_tree(prev.tree);
c4::print("CURR:"); print_tree(tree);
#endif
test_compare(prev.tree, tree);
}
_nfo_print_tree("PREV_", prev_.tree);
_nfo_print_tree("CURR", tree);
test_compare(prev_.tree, tree);
}
void compare_emitted(ProcLevel const& prev)
void compare_emitted(ProcLevel & prev_)
{
RYML_ASSERT(&prev_ == prev);
if(!prev_.was_emitted)
{
_nfo_logf("level[{}] not emitted. emit!", prev_.level);
prev_.emit();
}
if(!was_emitted)
{
_nfo_logf("level[{}] not emitted. emit!", level);
emit();
#if RYML_NFO
c4::log("PREV:\n{}", prev.emitted);
c4::log("CURR:\n{}", emitted);
#endif
EXPECT_EQ(emitted, prev.emitted);
}
_nfo_logf("level[{}]: EMITTED:\n{}", prev_.level, prev_.emitted);
_nfo_logf("level[{}]: EMITTED:\n{}", level, emitted);
if(emitted != prev_.emitted)
{
// workaround for lack of idempotency in tag normalization.
Tree from_prev = parse_in_arena(to_csubstr(prev_.emitted));
Tree from_this = parse_in_arena(to_csubstr(emitted));
from_prev.resolve_tags();
from_this.resolve_tags();
test_compare(from_prev, from_this);
}
}
};
@@ -260,8 +292,13 @@ struct Approach
casename = casename_;
filename = filename_;
allowed_failure = is_failure_expected(casename);
size_t level_index = 0;
ProcLevel *prev = nullptr;
for(ProcLevel &l : levels)
l.init(filename, src_, immutable_, reuse_);
{
l.init(level_index++, prev, filename, src_, immutable_, reuse_);
prev = &l;
}
expect_error = expect_error_;
}
@@ -289,7 +326,7 @@ struct Approach
for(size_t i = 1; i < num; ++i)
levels[i].compare_trees(levels[i-1]);
}
void compare_trees(size_t num, Approach const& other)
void compare_trees(size_t num, Approach & other)
{
if(allowed_failure)
GTEST_SKIP();
@@ -304,7 +341,7 @@ struct Approach
for(size_t i = 1; i < num; ++i)
levels[i].compare_emitted(levels[i-1]);
}
void compare_emitted(size_t num, Approach const& other)
void compare_emitted(size_t num, Approach & other)
{
if(allowed_failure)
GTEST_SKIP();
@@ -408,6 +445,9 @@ struct SuiteCase
using namespace c4;
using c4::to_csubstr;
if(to_csubstr(input_file) == "error")
input_file = "in.yaml";
case_title = to_csubstr(case_title_);
case_dir = to_csubstr(case_dir_);

1
test/test_suite/test_suite_events.cpp
View File

@@ -128,7 +128,6 @@ csubstr filtered_scalar(csubstr str, ScalarType scalar_type, Tree *tree)
return buf;
}
struct Scalar
{
OptionalScalar scalar = {};

37
test/test_suite/test_suite_events_emitter.cpp
View File

@@ -1,3 +1,6 @@
#ifndef RYML_SINGLE_HEADER
#include <c4/yml/std/string.hpp>
#endif
#include "test_suite_events.hpp"
namespace c4 {
@@ -7,9 +10,10 @@ struct EventsEmitter
{
substr buf;
size_t pos;
std::string tagbuf;
Tree const* C4_RESTRICT m_tree;
EventsEmitter(Tree const& tree, substr buf_) : buf(buf_), pos(), m_tree(&tree) {}
void emit_tag(csubstr tag);
void emit_tag(csubstr tag, size_t node);
void emit_scalar(csubstr val, bool quoted);
void emit_key_anchor_tag(size_t node);
void emit_val_anchor_tag(size_t node);
@@ -113,18 +117,31 @@ void EventsEmitter::emit_scalar(csubstr val, bool quoted)
pr(val.sub(prev)); // print remaining portion
}
void EventsEmitter::emit_tag(csubstr tag)
void EventsEmitter::emit_tag(csubstr tag, size_t node)
{
csubstr ntag = normalize_tag_long(tag);
if(ntag.begins_with('<'))
size_t tagsize = m_tree->resolve_tag(to_substr(tagbuf), tag, node);
if(tagsize)
{
pr(ntag);
if(tagsize > tagbuf.size())
{
tagbuf.resize(tagsize);
tagsize = m_tree->resolve_tag(to_substr(tagbuf), tag, node);
}
pr(to_substr(tagbuf).first(tagsize));
}
else
{
pr('<');
pr(ntag);
pr('>');
csubstr ntag = normalize_tag_long(tag);
if(ntag.begins_with('<'))
{
pr(ntag);
}
else
{
pr('<');
pr(ntag);
pr('>');
}
}
}
@@ -138,7 +155,7 @@ void EventsEmitter::emit_key_anchor_tag(size_t node)
if(m_tree->has_key_tag(node))
{
pr(' ');
emit_tag(m_tree->key_tag(node));
emit_tag(m_tree->key_tag(node), node);
}
}
@@ -152,7 +169,7 @@ void EventsEmitter::emit_val_anchor_tag(size_t node)
if(m_tree->has_val_tag(node))
{
pr(' ');
emit_tag(m_tree->val_tag(node));
emit_tag(m_tree->val_tag(node), node);
}
}

104
test/test_suite/test_suite_parts.cpp
View File

@@ -44,64 +44,73 @@ constexpr const AllowedFailure allowed_failures[] = {
//-------------------------------------------------------------------------
// SECTION 2. Expected errors that fail to materialize.
// maps
_("236B-error" , "should not accept final scalar in a map"),
_("2G84_00-error", "should not accept the block literal spec"),
_("2G84_01-error", "should not accept the block literal spec"),
_("3HFZ-error" , "should not accept scalar after ..."),
_("4EJS-error" , "should not accept double anchor for scalar"),
_("4JVG-error" , "should not accept double anchor for scalar"),
_("55WF-error" , "should not accept invalid escape in double quoted string"),
_("5LLU-error" , "should not accept folded scalar with wrong indented line after spaces only"),
_("5TRB-error" , "should not accept document-end marker in double quoted string"),
_("5U3A-error" , "should not accept opening a sequence on same line as map key"),
_("62EZ-error" , "should not accept invalid block mapping key on same line as previous key"),
_("6JTT-error" , "should not accept flow sequence without terminating ]"),
_("7LBH-error" , "should not accept multiline double quoted implicit keys"),
_("7MNF-error" , "should not accept final scalar in a map"),
_("8XDJ-error" , "should not accept comment in multiline scalar"),
_("9C9N-error" , "should not accept non-indented flow sequence"),
_("62EZ-error" , "should not accept invalid block mapping key on same line as previous key"),
_("9CWY-error" , "should not accept final scalar in a map"),
_("9HCY-error" , "should not accept tag directive in non-doc scope"),
_("CXX2-error" , "should not accept mapping with anchor on document start line"),
_("GDY7-error" , "should not accept comment that looks like a mapping key"),
_("D49Q-error" , "should not accept multiline single quoted implicit keys"),
_("DK4H-error" , "should not accept implicit key followed by newline"),
_("JY7Z-error" , "should not accept trailing content that looks like a mapping"),
_("SU74-error" , "should not accept anchor and alias as mapping key"),
_("T833-error" , "should not accept flow mapping missing a separating comma"),
_("VJP3_00-error", "should not accept flow collections over many lines"),
_("ZCZ6-error" , "should not accept invalid mapping in plain single line value"),
// seqs
_("5U3A-error" , "should not accept opening a sequence on same line as map key"),
_("6JTT-error" , "should not accept flow sequence without terminating ]"),
_("9C9N-error" , "should not accept non-indented flow sequence"),
_("9JBA-error" , "should not accept comment after flow seq terminating ]"),
_("9MAG-error" , "should not accept flow sequence with invalid comma at the beginning"),
_("CTN5-error" , "should not accept flow sequence with missing elements"),
_("CVW2-error" , "should not accept flow sequence with comment after ,"),
_("G5U8-error" , "should not accept [-, -]"),
_("KS4U-error" , "should not accept item after end of flow sequence"),
_("P2EQ-error" , "should not accept sequence item on same line as previous item"),
_("YJV2-error" , "should not accept [-]"),
// block scalars
_("2G84_00-error", "should not accept the block literal spec"),
_("2G84_01-error", "should not accept the block literal spec"),
_("5LLU-error" , "should not accept folded scalar with wrong indented line after spaces only"),
_("S4GJ-error" , "should not accept text after block scalar indicator"),
_("S98Z-error" , "should not accept block scalar with more spaces than first content line"),
_("X4QW-error" , "should not accept comment without whitespace after block scalar indicator"),
// quoted scalars
_("55WF-error" , "should not accept invalid escape in double quoted scalar"),
_("7LBH-error" , "should not accept multiline double quoted implicit keys"),
_("HRE5-error" , "should not accept double quoted scalar with escaped single quote"),
_("JKF3-error" , "should not accept multiline unindented double quoted scalar"),
_("QB6E-error" , "should not accept indented multiline quoted scalar"),
_("RXY3-error" , "should not accept document-end marker in single quoted string"),
_("SU5Z-error" , "should not accept comment without whitespace after double quoted scalar"),
// plain scalars
_("8XDJ-error" , "should not accept comment in multiline scalar"),
_("CML9-error" , "should not accept comment inside flow scalar"),
// documents/streams
_("3HFZ-error" , "should not accept scalar after ..."),
_("5TRB-error" , "should not accept document-end marker in double quoted string"),
_("9MMA-error" , "should not accept empty doc after %YAML directive"),
_("9MQT_01-error", "should not accept scalars after ..."),
_("B63P-error" , "should not accept directive without doc"),
_("BU8L-error" , "should not accept node properties spread over multiple lines"),
_("CML9-error" , "should not accept comment inside flow scalar"),
_("CTN5-error" , "should not accept flow sequence with missing elements"),
_("CVW2-error" , "should not accept flow sequence with comment after ,"),
_("CXX2-error" , "should not accept mapping with anchor on document start line"),
_("D49Q-error" , "should not accept multiline single quoted implicit keys"),
_("DK4H-error" , "should not accept implicit key followed by newline"),
_("EB22-error" , "should not accept missing document-end marker before directive"),
_("G5U8-error" , "should not accept [-, -]"),
_("GDY7-error" , "should not accept comment that looks like a mapping key"),
_("HRE5-error" , "should not accept double quoted scalar with escaped single quote"),
_("H7TQ-error" , "should not accept extra words after directive"),
_("JKF3-error" , "should not accept multiline unindented double quoted scalar"),
_("JY7Z-error" , "should not accept trailing content that looks like a mapping"),
_("KS4U-error" , "should not accept item after end of flow sequence"),
_("LHL4-error" , "should not accept tag"),
_("MUS6_00-error", "should not accept #... at the end of %YAML directive"),
_("MUS6_01-error", "should not accept #... at the end of %YAML directive"),
_("N782-error" , "should not accept document markers in flow style"),
_("P2EQ-error" , "should not accept sequence item on same line as previous item"),
_("QB6E-error" , "should not accept indented multiline quoted scalar"),
_("RHX7-error" , "should not accept directive without document end marker"),
_("RXY3-error" , "should not accept document-end marker in single quoted string"),
_("S4GJ-error" , "should not accept text after block scalar indicator"),
_("S98Z-error" , "should not accept block scalar with more spaces than first content line"),
_("SF5V-error" , "should not accept duplicate YAML directive"),
_("SU5Z-error" , "should not accept comment without whitespace after double quoted scalar"),
_("SU74-error" , "should not accept anchor and alias as mapping key"),
// anchors
_("4EJS-error" , "should not accept double anchor for scalar"),
_("4JVG-error" , "should not accept double anchor for scalar"),
_("SY6V-error" , "should not accept anchor before sequence entry on same line"),
_("T833-error" , "should not accept flow mapping missing a separating comma"),
// tags
_("9HCY-error" , "should not accept tag directive in non-doc scope"),
_("BU8L-error" , "should not accept node properties spread over multiple lines"),
_("LHL4-error" , "should not accept tag"),
_("U99R-error" , "should not accept comma in a tag"),
_("VJP3_00-error", "should not accept flow collections over many lines"),
_("X4QW-error" , "should not accept comment without whitespace after block scalar indicator"),
_("YJV2-error" , "should not accept [-]"),
_("ZCZ6-error" , "should not accept invalid mapping in plain single line value"),
_("QLJ7-error" , "tag directives should apply only to the next doc (?)"),
//-------------------------------------------------------------------------
@@ -110,7 +119,7 @@ constexpr const AllowedFailure allowed_failures[] = {
// These tests are skipped because they cover parts of YAML that
// are deliberately not implemented by ryml.
#ifndef RYML_WITH_TAB_TOKENS // - or : are supported only when the above macro is defined
#ifndef RYML_WITH_TAB_TOKENS // -<tab> or :<tab> are supported only when the above macro is defined
_("6BCT-in_yaml" , "tabs after - or :"),
_("J3BT-in_yaml-events" , "tabs after - or :"),
#endif
@@ -150,17 +159,6 @@ constexpr const AllowedFailure allowed_failures[] = {
_("XW4D-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"),
// anchors with : are not supported
_("2SXE-in_yaml-events" , "weird characters in anchors, anchors must not end with :"),
// tags are parsed as-is; tag lookup is not supported
_("5TYM-in_yaml-events" , "tag lookup is not supported"),
_("6CK3-in_yaml-events" , "tag lookup is not supported"),
_("6WLZ-in_yaml-events" , "tag lookup is not supported"),
_("9WXW-in_yaml-events" , "tag lookup is not supported"),
_("C4HZ-in_yaml-events" , "tag lookup is not supported"),
_("CC74-in_yaml-events" , "tag lookup is not supported"),
_("P76L-in_yaml-events" , "tag lookup is not supported"),
_("QLJ7-error" , "tag lookup is not supported"),
_("U3C3-in_yaml-events" , "tag lookup is not supported"),
_("Z9M4-in_yaml-events" , "tag lookup is not supported"),
// malformed json in the test spec
_("35KP-in_json" , "malformed JSON from multiple documents"),
_("5TYM-in_json" , "malformed JSON from multiple documents"),

325
test/test_tag_property.cpp
View File

@@ -4,6 +4,213 @@
namespace c4 {
namespace yml {
TEST(tag_directives, basic)
{
Tree t = parse_in_arena(R"(
%TAG !m! !my-
--- # Bulb here
!m!light fluorescent
...
%TAG !m! !meta-
--- # Color here
!m!light green
)");
EXPECT_EQ(t[0].val_tag(), "!m!light");
EXPECT_EQ(t[1].val_tag(), "!m!light");
EXPECT_EQ(t.num_tag_directives(), 2u);
char buf_[100];
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("<!my-light>"));
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("<!meta-light>"));
EXPECT_EQ(emitrs<std::string>(t), std::string(R"(%TAG !m! !my-
--- !m!light fluorescent
...
%TAG !m! !meta-
--- !m!light green
)"));
}
TEST(tag_directives, accepts_comment)
{
Tree t = parse_in_arena(R"(
%TAG !m! !my- # comment
--- # Bulb here
!m!light fluorescent
...
%TAG !m! !meta- # comment
--- # Color here
!m!light green
)");
EXPECT_EQ(t[0].val_tag(), "!m!light");
EXPECT_EQ(t[1].val_tag(), "!m!light");
EXPECT_EQ(t.num_tag_directives(), 2u);
char buf_[100];
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("<!my-light>"));
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("<!meta-light>"));
EXPECT_EQ(emitrs<std::string>(t), std::string(R"(%TAG !m! !my-
--- !m!light fluorescent
...
%TAG !m! !meta-
--- !m!light green
)"));
}
TEST(tag_directives, accepts_multiple_spaces)
{
Tree t = parse_in_arena(R"(
%TAG !m! !my- # comment
--- # Bulb here
!m!light fluorescent
...
%TAG !m! !meta- # comment
--- # Color here
!m!light green
)");
EXPECT_EQ(t[0].val_tag(), "!m!light");
EXPECT_EQ(t[1].val_tag(), "!m!light");
EXPECT_EQ(t.num_tag_directives(), 2u);
char buf_[100];
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("<!my-light>"));
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("<!meta-light>"));
EXPECT_EQ(emitrs<std::string>(t), std::string(R"(%TAG !m! !my-
--- !m!light fluorescent
...
%TAG !m! !meta-
--- !m!light green
)"));
}
TEST(tag_directives, errors)
{
{
Tree t;
ExpectError::do_check(&t, [&]{
t = parse_in_arena(R"(
%TAG
--- # Bulb here
!m!light fluorescent)");
});
}
{
Tree t;
ExpectError::do_check(&t, [&]{
t = parse_in_arena(R"(
%TAG !m!
--- # Bulb here
!m!light fluorescent)");
});
}
}
TEST(tag_directives, resolve_tags)
{
Tree t = parse_in_arena(R"(
%TAG !m! !my- # comment
--- # Bulb here
!m!light fluorescent: !m!light bulb
...
%TAG !m! !meta- # comment
--- # Color here
!m!light green: !m!light color
)");
EXPECT_EQ(t.docref(0)[0].key_tag(), "!m!light");
EXPECT_EQ(t.docref(0)[0].val_tag(), "!m!light");
EXPECT_EQ(t.num_tag_directives(), 2u);
t.resolve_tags();
EXPECT_EQ(t.docref(0)[0].key_tag(), "<!my-light>");
EXPECT_EQ(t.docref(0)[0].val_tag(), "<!my-light>");
EXPECT_EQ(emitrs<std::string>(t), std::string(R"(%TAG !m! !my-
---
!<!my-light> fluorescent: !<!my-light> bulb
...
%TAG !m! !meta-
---
!<!meta-light> green: !<!meta-light> color
)"));
}
TEST(tag_directives, safe_with_empty_tree)
{
Tree t;
t.resolve_tags();
EXPECT_TRUE(t.empty());
}
TEST(tag_directives, decode_uri_chars)
{
{
Tree t = parse_in_arena(R"(
%TAG !e! tag:example.com,2000:app/
---
- !e!%61%62%63%21 baz
)");
t.resolve_tags();
EXPECT_EQ(t.docref(0)[0].val_tag(), csubstr("<tag:example.com,2000:app/abc!>"));
}
{
Tree t;
auto checkerr = [&t](csubstr yaml){
ExpectError::do_check(&t, [&]{
t.clear();
t = parse_in_arena(yaml);
t.resolve_tags();
});
};
{
SCOPED_TRACE("without numbers at begin");
checkerr(R"(
%TAG !e! tag:example.com,2000:app/
---
- !e!%%62%63 baz
)");
}
{
SCOPED_TRACE("without numbers in the middle");
checkerr(R"(
%TAG !e! tag:example.com,2000:app/
---
- !e!%61%%63 baz
)");
}
{
SCOPED_TRACE("without numbers in the end");
checkerr(R"(
%TAG !e! tag:example.com,2000:app/
---
- !e!%61%62% baz
)");
}
{
SCOPED_TRACE("with wrong characters numbers at begin");
checkerr(R"(
%TAG !e! tag:example.com,2000:app/
---
- !e!%h%62%63 baz
)");
}
{
SCOPED_TRACE("with wrong characters in the middle");
checkerr(R"(
%TAG !e! tag:example.com,2000:app/
---
- !e!%61%hh%63 baz
)");
}
{
SCOPED_TRACE("with wrong characters in the end");
checkerr(R"(
%TAG !e! tag:example.com,2000:app/
---
- !e!%61%62%hh baz
)");
}
}
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
TEST(tags, test_suite_735Y)
{
csubstr yaml_without_seq = R"(
@@ -76,26 +283,26 @@ TEST(tags, parsing)
- !<!!str> val
- !<tag:yaml.org,2002:str> val
)");
EXPECT_EQ(t.rootref().val_tag(), "!!seq");
EXPECT_EQ(t[0].val_tag(), "!!map");
EXPECT_EQ(t[1].val_tag(), "!!map");
EXPECT_EQ(t[2].val_tag(), "!!seq");
EXPECT_EQ(t[0]["key1"].key_tag(), "!key");
EXPECT_EQ(t[0]["key1"].val_tag(), "!val");
EXPECT_EQ(t[0]["key2"].key_tag(), "!key");
EXPECT_EQ(t[0]["key2"].val_tag(), "!val");
EXPECT_EQ(t[0]["key3"].key_tag(), "<key>");
EXPECT_EQ(t[0]["key3"].val_tag(), "<val>");
EXPECT_EQ(t.rootref().val_tag(), csubstr("!!seq"));
EXPECT_EQ(t[0].val_tag(), csubstr("!!map"));
EXPECT_EQ(t[1].val_tag(), csubstr("!!map"));
EXPECT_EQ(t[2].val_tag(), csubstr("!!seq"));
EXPECT_EQ(t[0]["key1"].key_tag(), csubstr("!key"));
EXPECT_EQ(t[0]["key1"].val_tag(), csubstr("!val"));
EXPECT_EQ(t[0]["key2"].key_tag(), csubstr("<!key>"));
EXPECT_EQ(t[0]["key2"].val_tag(), csubstr("<!val>"));
EXPECT_EQ(t[0]["key3"].key_tag(), csubstr("<key>"));
EXPECT_EQ(t[0]["key3"].val_tag(), csubstr("<val>"));
EXPECT_EQ(t[0]["<!key> key4"].has_key_tag(), false);
EXPECT_EQ(t[0]["<!key> key4"].has_val_tag(), false);
EXPECT_EQ(t[0]["<!key> key4"].key(), "<!key> key4");
EXPECT_EQ(t[0]["<!key> key4"].val(), "<!val> val4");
EXPECT_EQ(t[2][5].val_tag(), "!!str");
EXPECT_EQ(t[0]["<!key> key4"].key(), csubstr("<!key> key4"));
EXPECT_EQ(t[0]["<!key> key4"].val(), csubstr("<!val> val4"));
EXPECT_EQ(t[2][5].val_tag(), csubstr("!!str"));
EXPECT_EQ(emitrs<std::string>(t), R"(!!seq
- !!map
!key key1: !val val1
!key key2: !val val2
!<!key> key2: !<!val> val2
!<key> key3: !<val> val3
<!key> key4: <!val> val4
- !!map
@@ -104,8 +311,8 @@ TEST(tags, parsing)
- !val val
- !str val
- <!str> val
- !str val
- !!str val
- !<!str> val
- !<!!str> val
- !!str val
)");
}
@@ -500,20 +707,20 @@ TEST(normalize_tag, basic)
EXPECT_EQ(normalize_tag("<tag:yaml.org,2002:timestamp>" ), "!!timestamp");
EXPECT_EQ(normalize_tag("<tag:yaml.org,2002:value>" ), "!!value");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:map>" ), "!!map");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:omap>" ), "!!omap");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:pairs>" ), "!!pairs");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:set>" ), "!!set");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:seq>" ), "!!seq");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:binary>" ), "!!binary");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:bool>" ), "!!bool");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:float>" ), "!!float");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:int>" ), "!!int");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:merge>" ), "!!merge");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:null>" ), "!!null");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:str>" ), "!!str");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:timestamp>" ), "!!timestamp");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:value>" ), "!!value");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:map>" ), "!!map");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:omap>" ), "!!omap");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:pairs>" ), "!!pairs");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:set>" ), "!!set");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:seq>" ), "!!seq");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:binary>" ), "!!binary");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:bool>" ), "!!bool");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:float>" ), "!!float");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:int>" ), "!!int");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:merge>" ), "!!merge");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:null>" ), "!!null");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:str>" ), "!!str");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:timestamp>"), "!!timestamp");
EXPECT_EQ(normalize_tag("!<tag:yaml.org,2002:value>" ), "!!value");
EXPECT_EQ(normalize_tag("!!map" ), "!!map");
EXPECT_EQ(normalize_tag("!!omap" ), "!!omap");
@@ -530,13 +737,17 @@ TEST(normalize_tag, basic)
EXPECT_EQ(normalize_tag("!!timestamp"), "!!timestamp");
EXPECT_EQ(normalize_tag("!!value" ), "!!value");
EXPECT_EQ(normalize_tag("<!foo>"), "!foo");
EXPECT_EQ(normalize_tag("<foo>"), "<foo>");
EXPECT_EQ(normalize_tag("<!>"), "!");
EXPECT_EQ(normalize_tag("!!foo" ), "!!foo");
EXPECT_EQ(normalize_tag("!<!foo>"), "!foo");
EXPECT_EQ(normalize_tag("!my-light"), "!my-light");
EXPECT_EQ(normalize_tag("!foo"), "!foo");
EXPECT_EQ(normalize_tag("<!foo>"), "<!foo>");
EXPECT_EQ(normalize_tag("<foo>"), "<foo>");
EXPECT_EQ(normalize_tag("<!>"), "<!>");
EXPECT_EQ(normalize_tag("!<!foo>"), "<!foo>");
EXPECT_EQ(normalize_tag("!<foo>"), "<foo>");
EXPECT_EQ(normalize_tag("!<!>"), "!");
EXPECT_EQ(normalize_tag("!<!>"), "<!>");
}
TEST(normalize_tag_long, basic)
@@ -559,20 +770,20 @@ TEST(normalize_tag_long, basic)
EXPECT_EQ(normalize_tag_long("<tag:yaml.org,2002:timestamp>" ), "<tag:yaml.org,2002:timestamp>");
EXPECT_EQ(normalize_tag_long("<tag:yaml.org,2002:value>" ), "<tag:yaml.org,2002:value>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:map>" ), "<tag:yaml.org,2002:map>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:omap>" ), "<tag:yaml.org,2002:omap>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:pairs>" ), "<tag:yaml.org,2002:pairs>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:set>" ), "<tag:yaml.org,2002:set>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:seq>" ), "<tag:yaml.org,2002:seq>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:binary>" ), "<tag:yaml.org,2002:binary>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:bool>" ), "<tag:yaml.org,2002:bool>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:float>" ), "<tag:yaml.org,2002:float>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:int>" ), "<tag:yaml.org,2002:int>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:merge>" ), "<tag:yaml.org,2002:merge>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:null>" ), "<tag:yaml.org,2002:null>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:str>" ), "<tag:yaml.org,2002:str>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:timestamp>" ), "<tag:yaml.org,2002:timestamp>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:value>" ), "<tag:yaml.org,2002:value>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:map>" ), "<tag:yaml.org,2002:map>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:omap>" ), "<tag:yaml.org,2002:omap>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:pairs>" ), "<tag:yaml.org,2002:pairs>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:set>" ), "<tag:yaml.org,2002:set>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:seq>" ), "<tag:yaml.org,2002:seq>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:binary>" ), "<tag:yaml.org,2002:binary>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:bool>" ), "<tag:yaml.org,2002:bool>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:float>" ), "<tag:yaml.org,2002:float>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:int>" ), "<tag:yaml.org,2002:int>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:merge>" ), "<tag:yaml.org,2002:merge>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:null>" ), "<tag:yaml.org,2002:null>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:str>" ), "<tag:yaml.org,2002:str>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:timestamp>"), "<tag:yaml.org,2002:timestamp>");
EXPECT_EQ(normalize_tag_long("!<tag:yaml.org,2002:value>" ), "<tag:yaml.org,2002:value>");
EXPECT_EQ(normalize_tag_long("!!map" ), "<tag:yaml.org,2002:map>");
EXPECT_EQ(normalize_tag_long("!!omap" ), "<tag:yaml.org,2002:omap>");
@@ -589,13 +800,19 @@ TEST(normalize_tag_long, basic)
EXPECT_EQ(normalize_tag_long("!!timestamp"), "<tag:yaml.org,2002:timestamp>");
EXPECT_EQ(normalize_tag_long("!!value" ), "<tag:yaml.org,2002:value>");
EXPECT_EQ(normalize_tag_long("<!foo>"), "!foo");
EXPECT_EQ(normalize_tag_long("<foo>"), "<foo>");
EXPECT_EQ(normalize_tag_long("<!>"), "!");
EXPECT_EQ(normalize_tag_long("!!foo" ), "!!foo");
EXPECT_EQ(normalize_tag_long("!<!foo>"), "!foo");
EXPECT_EQ(normalize_tag_long("!my-light"), "!my-light");
EXPECT_EQ(normalize_tag_long("!foo"), "!foo");
EXPECT_EQ(normalize_tag_long("<!foo>"), "<!foo>");
EXPECT_EQ(normalize_tag_long("<foo>"), "<foo>");
EXPECT_EQ(normalize_tag_long("<!>"), "<!>");
EXPECT_EQ(normalize_tag_long("!<!foo>"), "<!foo>");
EXPECT_EQ(normalize_tag_long("!<foo>"), "<foo>");
EXPECT_EQ(normalize_tag_long("!<!>"), "!");
EXPECT_EQ(normalize_tag_long("!<!foo>"), "<!foo>");
EXPECT_EQ(normalize_tag_long("!<foo>"), "<foo>");
EXPECT_EQ(normalize_tag_long("!<!>"), "<!>");
}

75
test/test_tree.cpp
View File

@@ -3528,7 +3528,6 @@ TEST(set_root_as_stream, root_is_docval)
}
//-------------------------------------------
//-------------------------------------------
//-------------------------------------------
@@ -3596,6 +3595,80 @@ doc4
}
//-------------------------------------------
//-------------------------------------------
//-------------------------------------------
TEST(Tree, add_tag_directives)
{
#if RYML_MAX_TAG_DIRECTIVES != 4
#error this test assumes RYML_MAX_TAG_DIRECTIVES == 4
#endif
const TagDirective td[RYML_MAX_TAG_DIRECTIVES + 1] = {
TagDirective{csubstr("!a!"), csubstr("!ay-"), 0u},
TagDirective{csubstr("!b!"), csubstr("!by-"), 0u},
TagDirective{csubstr("!c!"), csubstr("!cy-"), 0u},
TagDirective{csubstr("!d!"), csubstr("!dy-"), 0u},
TagDirective{csubstr("!e!"), csubstr("!ey-"), 0u},
};
Tree t;
auto check_up_to = [&](size_t num)
{
size_t pos = 0;
EXPECT_EQ(t.num_tag_directives(), num);
for(TagDirective const& d : t.tag_directives())
{
EXPECT_EQ(d.handle.str, td[pos].handle.str);
EXPECT_EQ(d.handle.len, td[pos].handle.len);
EXPECT_EQ(d.prefix.str, td[pos].prefix.str);
EXPECT_EQ(d.prefix.str, td[pos].prefix.str);
EXPECT_EQ(d.next_node_id, td[pos].next_node_id);
++pos;
}
EXPECT_EQ(pos, num);
};
check_up_to(0);
t.add_tag_directive(td[0]);
check_up_to(1);
t.add_tag_directive(td[1]);
check_up_to(2);
t.add_tag_directive(td[2]);
check_up_to(3);
t.add_tag_directive(td[3]);
check_up_to(4);
ExpectError::do_check(&t, [&]{ // number exceeded
t.add_tag_directive(td[4]);
});
t.clear_tag_directives();
check_up_to(0);
}
TEST(Tree, resolve_tag)
{
csubstr yaml = R"(
#%TAG !m! !my-
--- # Bulb here
!m!light fluorescent
...
#%TAG !m! !meta-
--- # Color here
!m!light green
)";
// we're not testing the parser here, just the tag mechanics.
// So we'll add the tag directives by hand.
Tree t = parse_in_arena(yaml);
EXPECT_EQ(t[0].val_tag(), "!m!light");
EXPECT_EQ(t[1].val_tag(), "!m!light");
EXPECT_EQ(t.num_tag_directives(), 0u);
t.add_tag_directive(TagDirective{csubstr("!m!"), csubstr("!my-"), 1});
t.add_tag_directive(TagDirective{csubstr("!m!"), csubstr("!meta-"), 2});
EXPECT_EQ(t.num_tag_directives(), 2u);
char buf_[100];
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("<!my-light>"));
EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("<!meta-light>"));
}
//-------------------------------------------
// this is needed to use the test case library
Case const* get_case(csubstr /*name*/)

164
test/test_yaml_events.cpp
View File

@@ -243,6 +243,170 @@ description:
}
TEST(events, tag_directives_6CK3)
{
test_evts(
R"(
%TAG !e! tag:example.com,2000:app/
---
- !local foo
- !!str bar
- !e!tag%21 baz
)",
R"(+STR
+DOC ---
+SEQ
=VAL <!local> :foo
=VAL <tag:yaml.org,2002:str> :bar
=VAL <tag:example.com,2000:app/tag!> :baz
-SEQ
-DOC
-STR
)");
}
TEST(events, tag_directives_6VLF)
{
test_evts(
R"(
%FOO bar baz # Should be ignored
# with a warning.
--- "foo"
)",
R"(+STR
+DOC ---
=VAL 'foo
-DOC
-STR
)");
}
TEST(events, tag_directives_6WLZ)
{
test_evts(
R"(
# Private
---
!foo "bar"
...
# Global
%TAG ! tag:example.com,2000:app/
---
!foo "bar"
)",
R"(+STR
+DOC ---
=VAL <!foo> 'bar
-DOC
+DOC ---
=VAL <tag:example.com,2000:app/foo> 'bar
-DOC
-STR
)");
}
TEST(events, tag_directives_9WXW)
{
test_evts(
R"(
# Private
#--- # note this is commented out
!foo "bar"
...
# Global
%TAG ! tag:example.com,2000:app/
---
!foo "bar"
)",
R"(+STR
+DOC ---
=VAL <!foo> 'bar
-DOC
+DOC ---
=VAL <tag:example.com,2000:app/foo> 'bar
-DOC
-STR
)");
}
TEST(events, tag_directives_7FWL)
{
test_evts(
R"(!<tag:yaml.org,2002:str> foo :
!<!bar> baz
)",
R"(+STR
+DOC
+MAP
=VAL <tag:yaml.org,2002:str> :foo
=VAL <!bar> :baz
-MAP
-DOC
-STR
)");
}
TEST(events, tag_directives_P76L)
{
test_evts(
R"(
%TAG !! tag:example.com,2000:app/
---
!!int 1 - 3 # Interval, not integer
)",
R"(+STR
+DOC ---
=VAL <tag:example.com,2000:app/int> :1 - 3
-DOC
-STR
)");
}
TEST(events, tag_directives_S4JQ)
{
test_evts(
R"(
- "12"
- 12
- ! 12
)",
R"(+STR
+DOC
+SEQ
=VAL '12
=VAL :12
=VAL <!> :12
-SEQ
-DOC
-STR
)");
}
TEST(events, tag_directives_lookup)
{
test_evts(
R"(
%TAG !m! !my-
--- # Bulb here
!m!light fluorescent
...
%TAG !m! !meta-
--- # Color here
!m!light green
)",
R"(+STR
+DOC ---
=VAL <!my-light> :fluorescent
-DOC
+DOC ---
=VAL <!meta-light> :green
-DOC
-STR
)");
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------

33
tools/test_suite/Dockerfile Normal file
View File

@@ -0,0 +1,33 @@
FROM alpine:latest
# to run:
# docker run --rm -it -v $PWD:/host alpine sh
# docker run --rm -it -v $PWD:/host -w /host -v /tmp/bash_history:/root/.bash_history yts-importing bash
#
RUN apk add build-base \
&& apk add \
bash \
curl \
fortune \
git \
jq \
perl \
perl-app-cpanminus \
tig \
vim \
wget \
python \
cmake \
ninja \
&& true
RUN cpanm -n \
boolean \
Capture::Tiny \
XXX \
YAML::PP \
&& true
ENV PYTHONPATH=/python/lib/python3.7/site-packages

18
tools/test_suite/run_test_suite.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/bin/bash
set -x
set -e
d=$1
[ "$d" == "" ] && d=.
if [ ! -d $d ] ; then
echo "$d is not a directory"
exit 1
fi
d=$(cd $d ; pwd) # get absolute path
cd $d/yaml-test-runtimes
make force build
cd $d/yaml-test-suite
make clean run-tests export
xsel -b <export.tsv