Mirrors/rapidyaml
0
0
Fork 0
mirror of https://github.com/biojppm/rapidyaml.git synced 2026-01-18 21:41:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #536 from biojppm/int_handler

Browse Source 
Add compliant event handler as extra, for use in other languages
This commit is contained in:
jpmag
2025-09-29 23:46:54 +01:00
committed by GitHub
parent 9a044c88b8 9a5289005c
commit f7970d5401
99 changed files with 5831 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows-in/bash/install-configure.bash vendored
View File

@@ -1,10 +0,0 @@
mkdir -p $BDIR
mkdir -p $IDIR
cmake -S ${{matrix.sdir}} -B $BDIR \
-DRYML_BUILD_TESTS=ON \
-DRYML_VALGRIND=OFF \
-DCMAKE_BUILD_TYPE=${{matrix.bt}} \
-DCMAKE_INSTALL_PREFIX=$IDIR \
${{matrix.gen}} \
${{matrix.vars}} \
${{matrix.commonvars}}

6
.github/workflows-in/bash/install-preinstall.bash vendored
View File

@@ -1,6 +0,0 @@
if [ "${{matrix.sdir}}" == "test/test_install" ] ; then
mkdir -p $BDIR-staging
cmake -S . -B $BDIR-staging -DCMAKE_INSTALL_PREFIX=$PDIR -DCMAKE_BUILD_TYPE=${{matrix.bt}} ${{matrix.gen}} ${{matrix.commonvars}}
cmake --build $BDIR-staging --config ${{matrix.bt}} --target ${{matrix.tgt}} -j
cmake --build $BDIR-staging --config ${{matrix.bt}} --target install
fi

174
.github/workflows-in/install.ys vendored
View File

@@ -23,124 +23,34 @@ jobs:
strategy:
fail-fast: false
matrix:
fp fl sh lcxx =:
-['find_package' 'find_library' 'singleheader' 'linux/libcxx']
t-t-i t-t-s =: -['test/test_install' 'test/test_singleheader']
include:: -[
{:name "$fp/linux", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'g++-13',
:gen '-DCMAKE_CXX_COMPILER=g++-13', :tgt 'all', :bt 'Release',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars nil}
{:name "$fp/linux", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'g++-13',
:gen '-DCMAKE_CXX_COMPILER=g++-13', :tgt 'all', :bt 'Debug',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars nil}
{:name "$fp/$lcxx", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'clang++-14',
:gen '-DCMAKE_CXX_COMPILER=clang++-14', :tgt 'all', :bt 'Release',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars '-DRYML_USE_LIBCXX=ON'}
{:name "$fp/$lcxx", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'clang++-14',
:gen '-DCMAKE_CXX_COMPILER=clang++-14', :tgt 'all', :bt 'Debug',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars '-DRYML_USE_LIBCXX=ON'}
{:name "$fp/macos", :sdir t-t-i, :os 'macos-13', :cxx 'xcode',
:gen '-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64',
:tgt 'ALL_BUILD', :bt 'Release',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars nil}
{:name "$fp/macos", :sdir t-t-i, :os 'macos-13', :cxx 'xcode',
:gen '-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64',
:tgt 'ALL_BUILD', :bt 'Debug',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars nil}
{:name "$fp/win", :sdir t-t-i, :os 'windows-2022', :cxx 'vs2022',
:gen "-G 'Visual Studio 17 2022' -A x64",
:tgt 'ALL_BUILD', :bt 'Release',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars nil}
{:name "$fp/win", :sdir t-t-i, :os 'windows-2022', :cxx 'vs2022',
:gen "-G 'Visual Studio 17 2022' -A x64",
:tgt 'ALL_BUILD', :bt 'Debug',
:vars '-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake
-DRYML_TEST_INSTALL_PACKAGE_MODE=ON',
:commonvars nil}
include:
{:name "$fl/linux", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'g++-13',
:gen '-DCMAKE_CXX_COMPILER=g++-13', :tgt 'all', :bt 'Release',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars nil}
{:name "$fl/linux", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'g++-13',
:gen '-DCMAKE_CXX_COMPILER=g++-13', :tgt 'all', :bt 'Debug',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars nil}
{:name "$fl/$lcxx", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'clang++-14',
:gen '-DCMAKE_CXX_COMPILER=clang++-14', :tgt 'all', :bt 'Release',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars '-DRYML_USE_LIBCXX=ON'}
{:name "$fl/$lcxx", :sdir t-t-i, :os 'ubuntu-24.04', :cxx 'clang++-14',
:gen '-DCMAKE_CXX_COMPILER=clang++-14', :tgt 'all', :bt 'Debug',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars '-DRYML_USE_LIBCXX=ON'}
{:name "$fl/macos", :sdir t-t-i, :os 'macos-13', :cxx 'xcode',
:gen '-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64',
:tgt 'ALL_BUILD', :bt 'Release',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars nil}
{:name "$fl/macos", :sdir t-t-i, :os 'macos-13', :cxx 'xcode',
:gen '-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64',
:tgt 'ALL_BUILD', :bt 'Debug',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars nil}
{:name "$fl/win", :sdir t-t-i, :os 'windows-2022', :cxx 'vs2022',
:gen "-G 'Visual Studio 17 2022' -A x64",
:tgt 'ALL_BUILD', :bt 'Release',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars nil}
{:name "$fl/win", :sdir t-t-i, :os 'windows-2022', :cxx 'vs2022',
:gen "-G 'Visual Studio 17 2022' -A x64",
:tgt 'ALL_BUILD', :bt 'Debug',
:vars '-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR
-DRYML_TEST_INSTALL_PACKAGE_MODE=OFF',
:commonvars nil}
- {name: find_package/linux, sdir: test/test_install, os: ubuntu-24.04, cxx: g++-13, gen: -DCMAKE_CXX_COMPILER=g++-13, tgt: all, bt: Release, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON}
- {name: find_package/linux, sdir: test/test_install, os: ubuntu-24.04, cxx: g++-13, gen: -DCMAKE_CXX_COMPILER=g++-13, tgt: all, bt: Debug, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON}
- {name: find_package/linux/libcxx, sdir: test/test_install, os: ubuntu-24.04, cxx: clang++-14, gen: -DCMAKE_CXX_COMPILER=clang++-14, tgt: all, bt: Release, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON, commonvars: -DRYML_USE_LIBCXX=ON}
- {name: find_package/linux/libcxx, sdir: test/test_install, os: ubuntu-24.04, cxx: clang++-14, gen: -DCMAKE_CXX_COMPILER=clang++-14, tgt: all, bt: Debug, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON, commonvars: -DRYML_USE_LIBCXX=ON}
- {name: find_package/macos, sdir: test/test_install, os: macos-13, cxx: xcode, gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64, tgt: ALL_BUILD, bt: Release, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON}
- {name: find_package/macos, sdir: test/test_install, os: macos-13, cxx: xcode, gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64, tgt: ALL_BUILD, bt: Debug, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON}
- {name: find_package/win, sdir: test/test_install, os: windows-2022, cxx: vs2022, gen: -G 'Visual Studio 17 2022' -A x64, tgt: ALL_BUILD, bt: Release, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DRYML_TEST_INSTALL_PACKAGE_MODE=ON}
- {name: find_package/win, sdir: test/test_install, os: windows-2022, cxx: vs2022, gen: -G 'Visual Studio 17 2022' -A x64, tgt: ALL_BUILD, bt: Debug, vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DRYML_TEST_INSTALL_PACKAGE_MODE=ON}
{:name "$sh/linux", :sdir t-t-s, :os 'ubuntu-24.04', :cxx 'g++-13',
:gen '-DCMAKE_CXX_COMPILER=g++-13',
:tgt 'all', :bt 'Release', :vars nil, :commonvars nil}
{:name "$sh/linux", :sdir t-t-s, :os 'ubuntu-24.04', :cxx 'g++-13',
:gen '-DCMAKE_CXX_COMPILER=g++-13',
:tgt 'all', :bt 'Debug', :vars nil, :commonvars nil}
{:name "$sh/$lcxx", :sdir t-t-s, :os 'ubuntu-24.04', :cxx 'clang++-14',
:gen '-DCMAKE_CXX_COMPILER=clang++-14', :tgt 'all', :bt 'Release',
:vars nil, :commonvars '-DRYML_USE_LIBCXX=ON'}
{:name "$sh/$lcxx", :sdir t-t-s, :os 'ubuntu-24.04', :cxx 'clang++-14',
:gen '-DCMAKE_CXX_COMPILER=clang++-14', :tgt 'all', :bt 'Debug',
:vars nil, :commonvars '-DRYML_USE_LIBCXX=ON'}
{:name "$sh/macos", :sdir t-t-s, :os 'macos-13', :cxx 'xcode',
:gen '-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64',
:tgt 'ALL_BUILD', :bt 'Release', :vars nil, :commonvars nil}
{:name "$sh/macos", :sdir t-t-s, :os 'macos-13', :cxx 'xcode',
:gen '-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64',
:tgt 'ALL_BUILD', :bt 'Debug', :vars nil, :commonvars nil}
{:name "$sh/win", :sdir t-t-s, :os 'windows-2022', :cxx 'vs2022',
:gen "-G 'Visual Studio 17 2022' -A x64",
:tgt 'ALL_BUILD', :bt 'Release', :vars nil, :commonvars nil}
{:name "$sh/win", :sdir t-t-s, :os 'windows-2022', :cxx 'vs2022',
:gen "-G 'Visual Studio 17 2022' -A x64",
:tgt 'ALL_BUILD', :bt 'Debug', :vars nil, :commonvars nil}]
- {name: find_library/linux, sdir: test/test_install, os: ubuntu-24.04, cxx: g++-13, gen: -DCMAKE_CXX_COMPILER=g++-13, tgt: all, bt: Release, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF}
- {name: find_library/linux, sdir: test/test_install, os: ubuntu-24.04, cxx: g++-13, gen: -DCMAKE_CXX_COMPILER=g++-13, tgt: all, bt: Debug, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF}
- {name: find_library/linux/libcxx, sdir: test/test_install, os: ubuntu-24.04, cxx: clang++-14, gen: -DCMAKE_CXX_COMPILER=clang++-14, tgt: all, bt: Release, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF, commonvars: -DRYML_USE_LIBCXX=ON}
- {name: find_library/linux/libcxx, sdir: test/test_install, os: ubuntu-24.04, cxx: clang++-14, gen: -DCMAKE_CXX_COMPILER=clang++-14, tgt: all, bt: Debug, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF, commonvars: -DRYML_USE_LIBCXX=ON}
- {name: find_library/macos, sdir: test/test_install, os: macos-13, cxx: xcode, gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64, tgt: ALL_BUILD, bt: Release, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF}
- {name: find_library/macos, sdir: test/test_install, os: macos-13, cxx: xcode, gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64, tgt: ALL_BUILD, bt: Debug, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF}
- {name: find_library/win, sdir: test/test_install, os: windows-2022, cxx: vs2022, gen: -G 'Visual Studio 17 2022' -A x64, tgt: ALL_BUILD, bt: Release, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF}
- {name: find_library/win, sdir: test/test_install, os: windows-2022, cxx: vs2022, gen: -G 'Visual Studio 17 2022' -A x64, tgt: ALL_BUILD, bt: Debug, vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF}
- {name: singleheader/linux, sdir: test/test_singleheader, os: ubuntu-24.04, cxx: g++-13, gen: -DCMAKE_CXX_COMPILER=g++-13, tgt: all, bt: Release}
- {name: singleheader/linux, sdir: test/test_singleheader, os: ubuntu-24.04, cxx: g++-13, gen: -DCMAKE_CXX_COMPILER=g++-13, tgt: all, bt: Debug}
- {name: singleheader/linux/libcxx, sdir: test/test_singleheader, os: ubuntu-24.04, cxx: clang++-14, gen: -DCMAKE_CXX_COMPILER=clang++-14, tgt: all, bt: Release, commonvars: -DRYML_USE_LIBCXX=ON}
- {name: singleheader/linux/libcxx, sdir: test/test_singleheader, os: ubuntu-24.04, cxx: clang++-14, gen: -DCMAKE_CXX_COMPILER=clang++-14, tgt: all, bt: Debug, commonvars: -DRYML_USE_LIBCXX=ON}
- {name: singleheader/macos, sdir: test/test_singleheader, os: macos-13, cxx: xcode, gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64, tgt: ALL_BUILD, bt: Release}
- {name: singleheader/macos, sdir: test/test_singleheader, os: macos-13, cxx: xcode, gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64, tgt: ALL_BUILD, bt: Debug}
- {name: singleheader/win, sdir: test/test_singleheader, os: windows-2022, cxx: vs2022, gen: -G 'Visual Studio 17 2022' -A x64, tgt: ALL_BUILD, bt: Release}
- {name: singleheader/win, sdir: test/test_singleheader, os: windows-2022, cxx: vs2022, gen: -G 'Visual Studio 17 2022' -A x64, tgt: ALL_BUILD, bt: Debug}
env:
CXX_: ${{matrix.cxx}}
@@ -162,16 +72,28 @@ jobs:
uses: actions/setup-python@v5
with: { python-version: 3.9 }
- name: preinstall
run:: bash('install-preinstall')
run: |
if [ "${{matrix.sdir}}" == "test/test_install" ] ; then
mkdir -p $BDIR-staging
cmake -S . -B $BDIR-staging -DCMAKE_INSTALL_PREFIX=$PDIR -DCMAKE_BUILD_TYPE=${{matrix.bt}} ${{matrix.gen}} ${{matrix.commonvars}}
cmake --build $BDIR-staging --config ${{matrix.bt}} --target ${{matrix.tgt}} -j
cmake --build $BDIR-staging --config ${{matrix.bt}} --target install
fi
- name: configure
run:: bash('install-configure')
run: |
mkdir -p $BDIR
mkdir -p $IDIR
cmake -S ${{matrix.sdir}} -B $BDIR \
-DRYML_BUILD_TESTS=ON \
-DRYML_VALGRIND=OFF \
-DCMAKE_BUILD_TYPE=${{matrix.bt}} \
-DCMAKE_INSTALL_PREFIX=$IDIR \
${{matrix.gen}} \
${{matrix.vars}} \
${{matrix.commonvars}}
- name: build
run: >
cmake --build $BDIR
--config ${{matrix.bt}}
--target ryml-test-build -j
run: |
cmake --build $BDIR --config ${{matrix.bt}} --target ryml-test-build -j
- name: run
run: >
cmake --build $BDIR
--config ${{matrix.bt}}
--target ryml-test-run
run: |
cmake --build $BDIR --config ${{matrix.bt}} --target ryml-test-run

3
.github/workflows-in/rarearchs.ys vendored
View File

@@ -3,6 +3,9 @@
:use common: :all
:: workflow-setup()
# NOTE: to debug with qemu and gdb:
# https://stackoverflow.com/questions/59906042/how-to-debug-mips-interactively
jobs:
xcompile:
:: setup-job('rarearchs' 'xcompile')

4
.github/workflows-in/release.ys vendored
View File

@@ -422,7 +422,7 @@ jobs:
assets_path: ./assets/
# PyPI (test)
- name: Publish python packages to test PyPI
uses: pypa/gh-action-pypi-publish@v1.4.2
uses: pypa/gh-action-pypi-publish@v1.13.0
with:
repository_url: https://test.pypi.org/legacy/
user: __token__
@@ -432,7 +432,7 @@ jobs:
# PyPI (production)
- name: Publish python packages to production PyPI
if: contains(github.ref, 'tags/v')
uses: pypa/gh-action-pypi-publish@v1.4.2
uses: pypa/gh-action-pypi-publish@v1.13.0
with:
user: __token__
password: ${{secrets.PYPI_TOKEN}}

6
.github/workflows-in/samples.ys vendored
View File

@@ -31,10 +31,16 @@ jobs:
run: source .github/setenv.sh && c4_show_info
- name: singleheader
run: cd samples/singleheader && ./run.sh $BT
- name: singleheader-ints
run: cd samples/singleheader-ints && ./run.sh $BT
- name: singleheaderlib-static
run: cd samples/singleheaderlib && ./run_static.sh $BT
- name: singleheaderlib-shared
run: cd samples/singleheaderlib && ./run_shared.sh $BT
- name: singleheaderlib-ints-static
run: cd samples/singleheaderlib-ints && ./run_static.sh $BT
- name: singleheaderlib-ints-shared
run: cd samples/singleheaderlib-ints && ./run_shared.sh $BT
- name: add_subdirectory
run: cd samples/add_subdirectory && ./run.sh $BT
- name: find_package

26
.github/workflows/install.yml vendored
View File

@@ -38,7 +38,6 @@ jobs:
tgt: all
bt: Release
vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON
commonvars: null
- name: find_package/linux
sdir: test/test_install
os: ubuntu-24.04
@@ -47,7 +46,6 @@ jobs:
tgt: all
bt: Debug
vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON
commonvars: null
- name: find_package/linux/libcxx
sdir: test/test_install
os: ubuntu-24.04
@@ -74,7 +72,6 @@ jobs:
tgt: ALL_BUILD
bt: Release
vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON
commonvars: null
- name: find_package/macos
sdir: test/test_install
os: macos-13
@@ -83,7 +80,6 @@ jobs:
tgt: ALL_BUILD
bt: Debug
vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON
commonvars: null
- name: find_package/win
sdir: test/test_install
os: windows-2022
@@ -92,7 +88,6 @@ jobs:
tgt: ALL_BUILD
bt: Release
vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DRYML_TEST_INSTALL_PACKAGE_MODE=ON
commonvars: null
- name: find_package/win
sdir: test/test_install
os: windows-2022
@@ -101,7 +96,6 @@ jobs:
tgt: ALL_BUILD
bt: Debug
vars: -Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DRYML_TEST_INSTALL_PACKAGE_MODE=ON
commonvars: null
- name: find_library/linux
sdir: test/test_install
os: ubuntu-24.04
@@ -110,7 +104,6 @@ jobs:
tgt: all
bt: Release
vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF
commonvars: null
- name: find_library/linux
sdir: test/test_install
os: ubuntu-24.04
@@ -119,7 +112,6 @@ jobs:
tgt: all
bt: Debug
vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF
commonvars: null
- name: find_library/linux/libcxx
sdir: test/test_install
os: ubuntu-24.04
@@ -146,7 +138,6 @@ jobs:
tgt: ALL_BUILD
bt: Release
vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF
commonvars: null
- name: find_library/macos
sdir: test/test_install
os: macos-13
@@ -155,7 +146,6 @@ jobs:
tgt: ALL_BUILD
bt: Debug
vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF
commonvars: null
- name: find_library/win
sdir: test/test_install
os: windows-2022
@@ -164,7 +154,6 @@ jobs:
tgt: ALL_BUILD
bt: Release
vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF
commonvars: null
- name: find_library/win
sdir: test/test_install
os: windows-2022
@@ -173,7 +162,6 @@ jobs:
tgt: ALL_BUILD
bt: Debug
vars: -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF
commonvars: null
- name: singleheader/linux
sdir: test/test_singleheader
os: ubuntu-24.04
@@ -181,8 +169,6 @@ jobs:
gen: -DCMAKE_CXX_COMPILER=g++-13
tgt: all
bt: Release
vars: null
commonvars: null
- name: singleheader/linux
sdir: test/test_singleheader
os: ubuntu-24.04
@@ -190,8 +176,6 @@ jobs:
gen: -DCMAKE_CXX_COMPILER=g++-13
tgt: all
bt: Debug
vars: null
commonvars: null
- name: singleheader/linux/libcxx
sdir: test/test_singleheader
os: ubuntu-24.04
@@ -199,7 +183,6 @@ jobs:
gen: -DCMAKE_CXX_COMPILER=clang++-14
tgt: all
bt: Release
vars: null
commonvars: -DRYML_USE_LIBCXX=ON
- name: singleheader/linux/libcxx
sdir: test/test_singleheader
@@ -208,7 +191,6 @@ jobs:
gen: -DCMAKE_CXX_COMPILER=clang++-14
tgt: all
bt: Debug
vars: null
commonvars: -DRYML_USE_LIBCXX=ON
- name: singleheader/macos
sdir: test/test_singleheader
@@ -217,8 +199,6 @@ jobs:
gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64
tgt: ALL_BUILD
bt: Release
vars: null
commonvars: null
- name: singleheader/macos
sdir: test/test_singleheader
os: macos-13
@@ -226,8 +206,6 @@ jobs:
gen: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64
tgt: ALL_BUILD
bt: Debug
vars: null
commonvars: null
- name: singleheader/win
sdir: test/test_singleheader
os: windows-2022
@@ -235,8 +213,6 @@ jobs:
gen: -G 'Visual Studio 17 2022' -A x64
tgt: ALL_BUILD
bt: Release
vars: null
commonvars: null
- name: singleheader/win
sdir: test/test_singleheader
os: windows-2022
@@ -244,8 +220,6 @@ jobs:
gen: -G 'Visual Studio 17 2022' -A x64
tgt: ALL_BUILD
bt: Debug
vars: null
commonvars: null
env:
CXX_: ${{matrix.cxx}}
BT: ${{matrix.bt}}

4
.github/workflows/release.yml vendored
View File

@@ -998,7 +998,7 @@ jobs:
release_id: ${{steps.create_release.outputs.id}}
assets_path: ./assets/
- name: Publish python packages to test PyPI
uses: pypa/gh-action-pypi-publish@v1.4.2
uses: pypa/gh-action-pypi-publish@v1.13.0
with:
repository_url: https://test.pypi.org/legacy/
user: __token__
@@ -1007,7 +1007,7 @@ jobs:
skip_existing: true
- name: Publish python packages to production PyPI
if: contains(github.ref, 'tags/v')
uses: pypa/gh-action-pypi-publish@v1.4.2
uses: pypa/gh-action-pypi-publish@v1.13.0
with:
user: __token__
password: ${{secrets.PYPI_TOKEN}}

6
.github/workflows/samples.yml vendored
View File

@@ -65,10 +65,16 @@ jobs:
run: source .github/setenv.sh && c4_show_info
- name: singleheader
run: cd samples/singleheader && ./run.sh $BT
- name: singleheader-ints
run: cd samples/singleheader-ints && ./run.sh $BT
- name: singleheaderlib-static
run: cd samples/singleheaderlib && ./run_static.sh $BT
- name: singleheaderlib-shared
run: cd samples/singleheaderlib && ./run_shared.sh $BT
- name: singleheaderlib-ints-static
run: cd samples/singleheaderlib-ints && ./run_static.sh $BT
- name: singleheaderlib-ints-shared
run: cd samples/singleheaderlib-ints && ./run_shared.sh $BT
- name: add_subdirectory
run: cd samples/add_subdirectory && ./run.sh $BT
- name: find_package

2
CMakeLists.txt
View File

@@ -37,7 +37,7 @@ c4_add_library(ryml
ryml.hpp
ryml_std.hpp
c4/yml/detail/checks.hpp
c4/yml/detail/parser_dbg.hpp
c4/yml/detail/dbgprint.hpp
c4/yml/detail/print.hpp
c4/yml/detail/stack.hpp
c4/yml/common.hpp

73
README.md
View File

@@ -457,21 +457,40 @@ you can also generate a customized file suiting your particular needs
```console
[user@host rapidyaml]$ python3 tools/amalgamate.py -h
usage: amalgamate.py [-h] [--c4core | --no-c4core] [--fastfloat | --no-fastfloat] [--stl | --no-stl] [output]
usage: amalgamate.py [-h] [--c4core | --no-c4core] [--fastfloat | --no-fastfloat]
[--stl | --no-stl]
[-e {tree,testsuite,int,all,none} [{tree,testsuite,int,all,none} ...]]
[output]
positional arguments:
output output file. defaults to stdout
output output file. defaults to stdout
optional arguments:
-h, --help show this help message and exit
--c4core amalgamate c4core together with ryml. this is the default.
--no-c4core amalgamate c4core together with ryml. the default is --c4core.
--fastfloat enable fastfloat library. this is the default.
--no-fastfloat enable fastfloat library. the default is --fastfloat.
--stl enable stl interop. this is the default.
--no-stl enable stl interop. the default is --stl.
options:
-h, --help show this help message and exit
--c4core amalgamate c4core together with ryml. this is the default.
--no-c4core amalgamate c4core together with ryml. the default is
--c4core.
--fastfloat enable fastfloat library. this is the default.
--no-fastfloat enable fastfloat library. the default is --fastfloat.
--stl enable stl interop. this is the default.
--no-stl enable stl interop. the default is --stl.
-e, --events {tree,testsuite,int,all,none} [{tree,testsuite,int,all,none} ...]
Specify which event handlers to include. Possible values
are: 'tree': (the default) enable the normal ryml event
handler to create the tree, and additionally the Tree, Node,
parser and emitter utilities; if this is not enabled, none
of these components will be included in the amalgamated
file. 'testsuite': enable the (extra) YAML test suite event
handler. 'int': enable the (extra) integer-based event
handler. 'all': enable all event handlers. 'none': disable
all event handlers. The default is tree.
```
Note that you can select which event handlers are to be included in the
amalgamated header. This is useful for example when using only the int
event parsing (withou the ryml tree/node/parse/emit) for a programming
language or special application.
The amalgamated header file contains all the function declarations and
definitions. To use it in the project, `#include` the header at will
in any header or source file in the project, but in one source file,
@@ -560,6 +579,8 @@ more about each sample:
|:-------------------|--------------------------|:-------------|:-------------|
| [`singleheader`](./samples/singleheader) | **yes**<br>ryml brought as a single header file,<br>not as a library | [`CMakeLists.txt`](./samples/singleheader/CMakeLists.txt) | [`run.sh`](./samples/singleheader/run.sh) |
| [`singleheaderlib`](./samples/singleheaderlib) | **yes**<br>ryml brought as a library<br>but from the single header file | [`CMakeLists.txt`](./samples/singleheaderlib/CMakeLists.txt) | [`run_shared.sh` (shared library)](./samples/singleheaderlib/run_shared.sh)<br> [`run_static.sh` (static library)](./samples/singleheaderlib/run_static.sh) |
| [`singleheader-ints`](./samples/singleheader-ints) | **yes**<br>ryml brought as a single header file,<br>not as a library | [`CMakeLists.txt`](./samples/singleheader-ints/CMakeLists.txt) | [`run.sh`](./samples/singleheader-ints/run.sh) |
| [`singleheaderlib`](./samples/singleheaderlib-ints) | **yes**<br>ryml brought as a library<br>but from the single header file | [`CMakeLists.txt`](./samples/singleheaderlib-ints/CMakeLists.txt) | [`run_shared.sh` (shared library)](./samples/singleheaderlib-ints/run_shared.sh)<br> [`run_static.sh` (static library)](./samples/singleheaderlib-ints/run_static.sh) |
| [`add_subdirectory`](./samples/add_subdirectory) | **yes** | [`CMakeLists.txt`](./samples/add_subdirectory/CMakeLists.txt) | [`run.sh`](./samples/add_subdirectory/run.sh) |
| [`fetch_content`](./samples/fetch_content) | **yes** | [`CMakeLists.txt`](./samples/fetch_content/CMakeLists.txt) | [`run.sh`](./samples/fetch_content/run.sh) |
| [`find_package`](./samples/find_package) | **no**<br>needs prior install or package | [`CMakeLists.txt`](./samples/find_package/CMakeLists.txt) | [`run.sh`](./samples/find_package/run.sh) |
@@ -611,9 +632,31 @@ versions). You can find out how to achieve this by looking at the
One of the aims of ryml is to provide an efficient YAML API for other
languages. JavaScript is fully available, and there is already a
cursory implementation for Python using only the low-level API. After
ironing out the general approach, other languages are likely to
follow (all of this is possible because we're using
[SWIG](http://www.swig.org/), which makes it easy to do so).
ironing out the general approach, other languages are likely to follow
suit.
### Event buffer int handler
Recently we added an alternative parser event handler (not part of the
library). This handler parses the YAML source into a linear buffer of
integers, which contains events encoded as bitmasks, interleaved with
strings encoded as an offset (from the beginning of the source buffer)
and length.
This handler is fully compliant (ie it can handle container keys,
unlike the ryml C++ tree), and is also 2x to 3x faster to parse.
This handler is meant to be used in other programming languages while
also minimizing speed-killing inter-language calls, creating a full
representation of the YAML tree that can be processed at once in the
target programming language.
You can find the int event handler in the [`src_extra` source
folder](https://github.com/biojppm/rapidyaml/tree/master/src_extra). See
its doxygen documentation for details on how to use it, and how to
process the event array.
### JavaScript
@@ -692,7 +735,7 @@ following situations:
* ryml's tree does NOT accept containers as map keys: keys stored in
the tree must always be scalars. HOWEVER, this is a limitation only
of the final tree. The event-based parse engine DOES parse container
keys, as it is is meant to be used by other programming languages to
keys, as it is meant to be used by other programming languages to
create their native data-structures, and it is fully tested and
fully conformant (other than the general error permissiveness noted
below).
@@ -723,7 +766,7 @@ following situations:
of the stream or documents (as per the standard), BOMs inside
scalars are ignored. The [standard mandates that they should be
quoted](https://yaml.org/spec/1.2.2/#52-character-encodings) when
emitted, this is not done.
emitted; this is not done.
* ryml tends to be on the permissive side in several cases where the
YAML standard dictates that there should be an error; in many of these
cases, ryml will tolerate the input. This may be good or bad, but in

10
bm/CMakeLists.txt
View File

@@ -119,8 +119,13 @@ set(RYML_SAJSON_INC_DIR ${sajson_dir}/include)
function(ryml_add_bm_comparison_exe name)
c4_add_executable(ryml-bm-${name}
SOURCES bm_common.hpp ${ARGN}
../src_extra/c4/yml/extra/event_handler_ints.hpp
../src_extra/c4/yml/extra/event_handler_ints.cpp
LIBS ryml yaml yaml-cpp benchmark jsoncpp_static nlohmann_json c4fs
INC_DIRS ${RYML_RAPIDJSON_INC_DIR} ${RYML_SAJSON_INC_DIR}
INC_DIRS
${RYML_RAPIDJSON_INC_DIR}
${RYML_SAJSON_INC_DIR}
../src_extra
FOLDER bm)
if(RYML_DBG)
target_compile_definitions(ryml-bm-${name} PRIVATE RYML_DBG)
@@ -133,9 +138,8 @@ function(ryml_add_bm_comparison_exe name)
_c4_set_target_folder(ryml-bm-${name}-all bm/run)
endfunction()
ryml_add_bm_comparison_exe(parse bm_parse.cpp)
ryml_add_bm_comparison_exe(emit bm_emit.cpp)
ryml_add_bm_comparison_exe(parse bm_parse.cpp)
function(ryml_add_bm_comparison_case target name case_file)
c4_dbg("adding benchmark case: ${case_file}")

77
bm/bm_common.hpp
View File

@@ -4,6 +4,7 @@
#include <ryml_std.hpp>
#include <c4/fs/fs.hpp>
#include "c4/yml/parse.hpp"
#include "c4/yml/extra/event_handler_ints.hpp"
#include <vector>
#include <iostream>
@@ -82,14 +83,81 @@ namespace bm = benchmark;
# pragma clang diagnostic ignored "-Wdeprecated-declarations"
# pragma clang diagnostic ignored "-Wunused-variable"
# pragma clang diagnostic ignored "-Wsign-conversion"
# pragma clang diagnostic ignored "-Wold-style-cast"
#elif defined(__GNUC__)
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
# pragma GCC diagnostic ignored "-Wunused-variable"
# pragma GCC diagnostic ignored "-Wsign-conversion"
# pragma GCC diagnostic ignored "-Wold-style-cast"
#endif
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
using IntHandler = ryml::extra::EventHandlerInts;
using IntParser = ryml::ParseEngine<IntHandler>;
struct IntData
{
std::vector<int> events;
std::vector<char> arena;
void resize(ryml::extra::EventHandlerInts const& handler)
{
resize((size_t)handler.required_size_events(),
handler.required_size_arena());
}
void resize(ryml::substr src)
{
resize((size_t)ryml::extra::estimate_events_ints_size(src),
src.len);
}
void resize(size_t evt_sz, size_t arn_sz)
{
events.resize(evt_sz);
events.resize(arn_sz);
}
};
struct IntObjects
{
IntObjects(ryml::ParserOptions opts={})
: handler()
, parser(&handler, opts)
, data()
{
}
IntHandler handler;
IntParser parser;
IntData data;
bool again()
{
if(!handler.fits_buffers())
{
data.resize(handler);
return true;
}
return false;
}
};
inline void parse_yaml_inplace(ryml::csubstr filename, ryml::substr src, IntParser &parser, IntData *dst)
{
parser.m_evt_handler->reset(src,
ryml::to_substr(dst->arena),
dst->events.data(),
(int)dst->events.size());
parser.parse_in_place_ev(filename, src);
}
inline void parse_json_inplace(ryml::csubstr filename, ryml::substr src, IntParser &parser, IntData *dst)
{
parser.m_evt_handler->reset(src,
ryml::to_substr(dst->arena),
dst->events.data(),
(int)dst->events.size());
parser.parse_json_in_place_ev(filename, src);
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
@@ -99,6 +167,7 @@ enum : int {
kClearTreeArena=2,
kResetInPlace=4,
kReserveTree=8,
kReserveInts=8,
kAll=kClearTree|kClearTreeArena|kResetInPlace|kReserveTree,
};
@@ -117,6 +186,9 @@ struct BmCase
rapidjson::Document rapidjson_doc;
ryml::Tree libyaml_tree;
IntObjects int_obj;
IntObjects int_obj_nofilter{ryml::ParserOptions().scalar_filtering(false)};
void run(std::string name, int argc, char **argv)
{
bm_name = name;
@@ -173,6 +245,11 @@ USAGE: bm <case.yml>
RYML_CHECK(capacity > 0);
ryml_tree.reserve(capacity);
}
if(what & kReserveTree)
{
RYML_CHECK(capacity > 0);
ryml_tree.reserve(capacity);
}
if(what & kResetInPlace)
{
C4_ASSERT(in_place.size() == src.size());

243
bm/bm_parse.cpp
View File

@@ -279,6 +279,168 @@ void bm_ryml_json_arena_nofilter_reserve(bm::State& st)
}
//-----------------------------------------------------------------------------
void bm_rymlints_estimate(bm::State& st)
{
c4::csubstr src = c4::to_csubstr(s_bm_case->in_place).trimr('\0');
for(auto _ : st)
{
auto sz = c4::yml::extra::estimate_events_ints_size(src);
bm::DoNotOptimize(sz);
}
s_bm_case->report(st);
}
void bm_rymlints_yaml_inplace(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
for(auto _ : st)
{
IntObjects obj;
s_bm_case->prepare(st, kResetInPlace);
parse_yaml_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
if(obj.again())
{
s_bm_case->prepare(st, kResetInPlace);
parse_yaml_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
}
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
void bm_rymlints_yaml_inplace_reserve(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
int sz = ryml::extra::estimate_events_ints_size(src);
for(auto _ : st)
{
st.PauseTiming();
s_bm_case->prepare(kResetInPlace);
IntObjects obj;
obj.data.resize(src);
st.ResumeTiming();
parse_yaml_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
void bm_rymlints_yaml_inplace_nofilter(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
for(auto _ : st)
{
IntObjects obj(ryml::ParserOptions().scalar_filtering(false));
s_bm_case->prepare(st, kResetInPlace);
parse_yaml_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
if(obj.again())
{
s_bm_case->prepare(st, kResetInPlace);
parse_yaml_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
}
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
void bm_rymlints_yaml_inplace_nofilter_reserve(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
int sz = ryml::extra::estimate_events_ints_size(src);
for(auto _ : st)
{
st.PauseTiming();
s_bm_case->prepare(kResetInPlace);
IntObjects obj(ryml::ParserOptions().scalar_filtering(false));
obj.data.resize(src);
st.ResumeTiming();
parse_yaml_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
//-----------------------------------------------------------------------------
void bm_rymlints_json_inplace(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
for(auto _ : st)
{
ONLY_FOR_JSON;
IntObjects obj;
s_bm_case->prepare(st, kResetInPlace);
parse_json_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
if(obj.again())
{
s_bm_case->prepare(st, kResetInPlace);
parse_json_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
}
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
void bm_rymlints_json_inplace_reserve(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
int sz = ryml::extra::estimate_events_ints_size(src);
for(auto _ : st)
{
ONLY_FOR_JSON;
st.PauseTiming();
s_bm_case->prepare(kResetInPlace);
IntObjects obj;
obj.data.resize(src);
st.ResumeTiming();
parse_json_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
void bm_rymlints_json_inplace_nofilter(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
for(auto _ : st)
{
ONLY_FOR_JSON;
IntObjects obj(ryml::ParserOptions().scalar_filtering(false));
s_bm_case->prepare(st, kResetInPlace);
parse_json_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
if(obj.again())
{
s_bm_case->prepare(st, kResetInPlace);
parse_json_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
}
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
void bm_rymlints_json_inplace_nofilter_reserve(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
int sz = ryml::extra::estimate_events_ints_size(src);
for(auto _ : st)
{
ONLY_FOR_JSON;
st.PauseTiming();
s_bm_case->prepare(kResetInPlace);
IntObjects obj(ryml::ParserOptions().scalar_filtering(false));
obj.data.resize(src);
st.ResumeTiming();
parse_json_inplace(s_bm_case->filename, src, obj.parser, &obj.data);
bm::DoNotOptimize(obj);
}
s_bm_case->report(st);
}
//-----------------------------------------------------------------------------
void bm_ryml_yaml_inplace(bm::State& st)
@@ -513,6 +675,66 @@ void bm_ryml_json_arena_reuse_nofilter_reserve(bm::State& st)
}
//-----------------------------------------------------------------------------
void bm_rymlints_yaml_inplace_reuse(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
s_bm_case->int_obj.data.resize(src);
for(auto _ : st)
{
s_bm_case->prepare(st, kResetInPlace);
parse_yaml_inplace(s_bm_case->filename, src, s_bm_case->int_obj.parser, &s_bm_case->int_obj.data);
bm::DoNotOptimize(s_bm_case->int_obj);
}
s_bm_case->report(st);
}
void bm_rymlints_yaml_inplace_reuse_nofilter(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
s_bm_case->int_obj_nofilter.data.resize(src);
for(auto _ : st)
{
s_bm_case->prepare(st, kResetInPlace);
parse_yaml_inplace(s_bm_case->filename, src, s_bm_case->int_obj_nofilter.parser, &s_bm_case->int_obj_nofilter.data);
bm::DoNotOptimize(s_bm_case->int_obj_nofilter);
}
s_bm_case->report(st);
}
//-----------------------------------------------------------------------------
void bm_rymlints_json_inplace_reuse(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
s_bm_case->int_obj.data.resize(src);
for(auto _ : st)
{
ONLY_FOR_JSON;
s_bm_case->prepare(st, kResetInPlace);
parse_json_inplace(s_bm_case->filename, src, s_bm_case->int_obj.parser, &s_bm_case->int_obj.data);
bm::DoNotOptimize(s_bm_case->int_obj);
}
s_bm_case->report(st);
}
void bm_rymlints_json_inplace_reuse_nofilter(bm::State& st)
{
c4::substr src = c4::to_substr(s_bm_case->in_place).trimr('\0');
s_bm_case->int_obj_nofilter.data.resize(src);
for(auto _ : st)
{
ONLY_FOR_JSON;
s_bm_case->prepare(st, kResetInPlace);
parse_json_inplace(s_bm_case->filename, src, s_bm_case->int_obj_nofilter.parser, &s_bm_case->int_obj_nofilter.data);
bm::DoNotOptimize(s_bm_case->int_obj_nofilter);
}
s_bm_case->report(st);
}
//-----------------------------------------------------------------------------
void bm_ryml_yaml_inplace_reuse(bm::State& st)
@@ -617,6 +839,16 @@ void bm_ryml_json_inplace_reuse_nofilter_reserve(bm::State& st)
//-----------------------------------------------------------------------------
BENCHMARK(bm_rymlints_estimate);
BENCHMARK(bm_rymlints_json_inplace_reuse_nofilter);
BENCHMARK(bm_rymlints_json_inplace_reuse);
BENCHMARK(bm_rymlints_yaml_inplace_reuse_nofilter);
BENCHMARK(bm_rymlints_yaml_inplace_reuse);
BENCHMARK(bm_ryml_json_inplace_reuse_nofilter_reserve);
BENCHMARK(bm_ryml_json_inplace_reuse_nofilter);
BENCHMARK(bm_ryml_json_inplace_reuse_reserve);
@@ -639,6 +871,17 @@ BENCHMARK(bm_ryml_yaml_arena_reuse_reserve);
BENCHMARK(bm_ryml_yaml_arena_reuse);
BENCHMARK(bm_rymlints_json_inplace_nofilter_reserve);
BENCHMARK(bm_rymlints_json_inplace_nofilter);
BENCHMARK(bm_rymlints_json_inplace_reserve);
BENCHMARK(bm_rymlints_json_inplace);
BENCHMARK(bm_rymlints_yaml_inplace_nofilter_reserve);
BENCHMARK(bm_rymlints_yaml_inplace_nofilter);
BENCHMARK(bm_rymlints_yaml_inplace_reserve);
BENCHMARK(bm_rymlints_yaml_inplace);
BENCHMARK(bm_ryml_json_inplace_nofilter_reserve);
BENCHMARK(bm_ryml_json_inplace_nofilter);
BENCHMARK(bm_ryml_json_inplace_reserve);

13
changelog/current.md
View File

@@ -1,3 +1,16 @@
### Extra event handlers
[PR#536](https://github.com/biojppm/rapidyaml/pull/536) adds a new major extra feature: a parser event handler that creates a compact representation of the YAML tree in a buffer of integers containing masks (to represent events) and offset+length (to represent strings in the source buffer).
This handler is meant for use by other programming languages, and it supports container keys (unlike the ryml tree). You can find this handler among the other headers in the [new `src_extra` folder](https://github.com/biojppm/rapidyaml/tree/master/src_extra).
### Changes
- In [PR#536](https://github.com/biojppm/rapidyaml/pull/536) the location functions were moved from `ParserEngine` to `Tree` and `ConstNodeRef`. The parser engine is now fully agnostic vis-a-vis the type of the event-handler. (The location functions in the parser engine were a legacy of the initial implementation of the parser which was meant to create only ryml trees).
- The tool ryml-yaml-events was updated to also dump integer events (and its command line options were changed to enable the different choices).
### Fixes
- Fix [#524](https://github.com/biojppm/rapidyaml/issues/524) ([PR#525](https://github.com/biojppm/rapidyaml/pull/525)): problem parsing nested map value in complex map. Kudos to @MatthewSteel!

3
doc/Doxyfile
View File

@@ -952,8 +952,7 @@ WARN_LOGFILE =
INPUT = \
./doxy_main.md \
../src \
../test/test_suite/test_suite_event_handler.hpp \
../test/test_suite/test_suite_event_handler.cpp \
../src_extra \
../samples/quickstart.cpp \
../ext/c4core/src/c4/substr.hpp \
../ext/c4core/src/c4/charconv.hpp \

2
doc/Makefile
View File

@@ -25,7 +25,7 @@ help:
cp -favr doxygen/html "$(BUILDDIR)/html/doxygen"
doxy:
# ensure submodules are checked out
@# ensure submodules are checked out
if [ ! -f $(shell pwd)/../ext/c4core/cmake/c4Project.cmake ] ; then echo "ERROR: submodules not present" ; exit 1 ; fi
if [ ! -f $(shell pwd)/../ext/c4core/src/c4/charconv.hpp ] ; then echo "ERROR: submodules not present" ; exit 1 ; fi
doxygen Doxyfile

34
doc/sphinx_other_languages.rst
View File

@@ -2,11 +2,35 @@ Other languages
===============
One of the aims of ryml is to provide an efficient YAML API for other
languages. JavaScript is fully available, and there is already a cursory
implementation for Python using only the low-level API. After ironing
out the general approach, other languages are likely to follow (all of
this is possible because we’re using `SWIG <http://www.swig.org/>`__,
which makes it easy to do so).
languages. JavaScript is fully available, and there is already a
cursory implementation for Python using only the low-level API. After
ironing out the general approach, other languages are likely to follow
suit.
Event buffer int handler
------------------------
Recently we added an alternative parser event handler (not part of the
library). This handler parses the YAML source into a linear buffer of
integers, which contains events encoded as bitmasks, interleaved with
strings encoded as an offset (from the beginning of the source buffer)
and length.
This handler is fully compliant (ie it can handle container keys,
unlike the ryml C++ tree), and is also 2x to 3x faster to parse.
This handler is meant to be used in other programming languages while
also minimizing speed-killing inter-language calls, creating a full
representation of the YAML tree that can be processed at once in the
target programming language.
You can find the int event handler in the `src_extra source folder
<https://github.com/biojppm/rapidyaml/tree/master/src_extra>`__. See
its doxygen documentation for details on how to use it, and how to
process the event array.
JavaScript

88
doc/sphinx_using.rst
View File

@@ -38,24 +38,32 @@ Windows via Git Bash or the WSL). Click on the links below to find out
more about each sample:
+-------------------------------------------------------------------------------------------------+----------------------------------+--------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+
| Sample name | ryml is part of build? | cmake file | commands |
+=================================================================================================+==================================+==============================================================================================================+=============================================================================================================+
| `singleheader <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader>`_ | | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader/run.sh>`_ |
| | | ryml brought as a single | | |
| | | header, not as a library | | |
+-------------------------------------------------------------------------------------------------+----------------------------------+--------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+
| `singleheaderlib <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib>`_ | | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib/CMakeLists.txt>`_ | | `run_shared.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib/run_shared.sh>`_ |
| | | ryml brought as library | | | `run_static.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib/run_static.sh>`_ |
| | | but from the single header | | |
+-------------------------------------------------------------------------------------------------+----------------------------------+--------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+
| `add_subdirectory <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/add_subdirectory>`_ | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/add_subdirectory/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/add_subdirectory/run.sh>`_ |
+-------------------------------------------------------------------------------------------------+----------------------------------+--------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+
| `fetch_content <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/fetch_content>`_ | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/fetch_content/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/fetch_content/run.sh>`_ |
+-------------------------------------------------------------------------------------------------+----------------------------------+--------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+
| `find_package <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/find_package>`_ | | **no** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/find_package/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/find_package/run.sh>`_ |
| | | needs prior install or package | | |
+-------------------------------------------------------------------------------------------------+----------------------------------+--------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
| Sample name | ryml is part of build? | cmake file | commands |
+==========================================================================================================+==================================+===================================================================================================================+==================================================================================================================+
| `singleheader <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader>`_ | | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader/run.sh>`_ |
| | | ryml brought as a single | | |
| | | header, not as a library | | |
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
| `singleheaderlib <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib>`_ | | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib/CMakeLists.txt>`_ | | `run_shared.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib/run_shared.sh>`_ |
| | | ryml brought as library | | | `run_static.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib/run_static.sh>`_ |
| | | but from the single header | | |
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
| `singleheader-ints <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader-ints>`_ | | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader-ints/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheader-ints/run.sh>`_ |
| | | ryml brought as a single | | |
| | | header, not as a library | | |
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
| `singleheaderlib-ints <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib-ints>`_ | | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib-ints/CMakeLists.txt>`_ | | `run_shared.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib-ints/run_shared.sh>`_ |
| | | ryml brought as library | | | `run_static.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/singleheaderlib-ints/run_static.sh>`_ |
| | | but from the single header | | |
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
| `add_subdirectory <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/add_subdirectory>`_ | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/add_subdirectory/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/add_subdirectory/run.sh>`_ |
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
| `fetch_content <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/fetch_content>`_ | **yes** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/fetch_content/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/fetch_content/run.sh>`_ |
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
| `find_package <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/find_package>`_ | | **no** | `CMakeLists.txt <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/find_package/CMakeLists.txt>`_ | `run.sh <https://github.com/biojppm/rapidyaml/blob/v0.9.0/samples/find_package/run.sh>`_ |
| | | needs prior install or package | | |
+----------------------------------------------------------------------------------------------------------+----------------------------------+-------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
As a single-header
@@ -72,20 +80,38 @@ file suiting your particular needs (or commit):
.. code-block:: console
[user@host rapidyaml]$ python3 tools/amalgamate.py -h
usage: amalgamate.py [-h] [--c4core | --no-c4core] [--fastfloat | --no-fastfloat] [--stl | --no-stl] [output]
positional arguments:
output output file. defaults to stdout
optional arguments:
-h, --help show this help message and exit
--c4core amalgamate c4core together with ryml. this is the default.
--no-c4core amalgamate c4core together with ryml. the default is --c4core.
--fastfloat enable fastfloat library. this is the default.
--no-fastfloat enable fastfloat library. the default is --fastfloat.
--stl enable stl interop. this is the default.
--no-stl enable stl interop. the default is --stl.
usage: amalgamate.py [-h] [--c4core | --no-c4core] [--fastfloat | --no-fastfloat]
[--stl | --no-stl]
[-e {tree,testsuite,int,all,none} [{tree,testsuite,int,all,none} ...]]
[output]
positional arguments:
output output file. defaults to stdout
options:
-h, --help show this help message and exit
--c4core amalgamate c4core together with ryml. this is the default.
--no-c4core amalgamate c4core together with ryml. the default is
--c4core.
--fastfloat enable fastfloat library. this is the default.
--no-fastfloat enable fastfloat library. the default is --fastfloat.
--stl enable stl interop. this is the default.
--no-stl enable stl interop. the default is --stl.
-e, --events {tree,testsuite,int,all,none} [{tree,testsuite,int,all,none} ...]
Specify which event handlers to include. Possible values
are: 'tree': (the default) enable the normal ryml event
handler to create the tree, and additionally the Tree, Node,
parser and emitter utilities; if this is not enabled, none
of these components will be included in the amalgamated
file. 'testsuite': enable the (extra) YAML test suite event
handler. 'int': enable the (extra) integer-based event
handler. 'all': enable all event handlers. 'none': disable
all event handlers. The default is tree.
Note that you can select which event handlers are to be included in the
amalgamated header. This is useful for example when using only the int
event parsing (withou the ryml tree/node/parse/emit) for a programming
language or special application.
The amalgamated header file contains all the function declarations and
definitions. To use it in the project, ``#include`` the header at will

2
ext/c4core

Submodule ext/c4core updated: 95a69ad352...b3ce4e2951

159
samples/quickstart-ints.cpp Normal file
View File

@@ -0,0 +1,159 @@
// This file shows a quick example of parsing YAML to an int events
// buffer. Since this functionality is meant to implement in other
// programming languages, the code is kept very simple, and using only
// C-like idioms.
// ryml can be used as a single header, or as a simple library:
#if defined(RYML_SINGLE_HEADER) // using the single header directly in the executable
#define RYML_SINGLE_HDR_DEFINE_NOW
#ifndef RYML_SINGLE_HEADER_INTS
#include <ryml_all.hpp>
#else
#include <ryml_ints.hpp>
#endif
#elif defined(RYML_SINGLE_HEADER_LIB) // using the single header from a library
#ifndef RYML_SINGLE_HEADER_INTS
#include <ryml_all.hpp>
#else
#include <ryml_ints.hpp>
#endif
#else
#include <c4/yml/parse_engine.def.hpp>
#endif
#ifndef RYML_SINGLE_HEADER_INTS
#include <c4/yml/extra/event_handler_ints.hpp>
#endif
// NOLINTBEGIN(hicpp-signed-bitwise)
int main(int, const char *[])
{
using namespace c4::yml::extra::ievt;
auto PSTR_ = c4::yml::extra::ievt::PSTR; // PSTR does not work in windows
// YAML code to be parsed in place
char yaml[] = "do: a deer, a female deer\n"
"re: a drop of golden sun\n"
"mi: a name I call myself\n"
"fa: a long long way to run\n";
// these are the event values we expect
const int expected_events[] = {
BSTR,
BDOC,
VAL_|BMAP|BLCK,
//
KEY_|SCLR|PLAI, 0, 2, // "do"
VAL_|SCLR|PLAI|PSTR_, 4, 21, // "a deer, a female deer"
//
KEY_|SCLR|PLAI|PSTR_, 26, 2, // "re"
VAL_|SCLR|PLAI|PSTR_, 30, 20, // "a drop of golden sun"
//
KEY_|SCLR|PLAI|PSTR_, 51, 2, // "mi"
VAL_|SCLR|PLAI|PSTR_, 55, 20, // "a name I call myself"
//
KEY_|SCLR|PLAI|PSTR_, 76, 2, // "fa"
VAL_|SCLR|PLAI|PSTR_, 80, 22, // "a long long way to run"
//
EMAP|PSTR_,
EDOC,
ESTR,
};
/* the output should be this:
*
* success! YAML requires event size 30, estimated=49
* pos=0 event[0]: 0x1
* pos=1 event[1]: 0x4
* pos=2 event[2]: 0x110010
* pos=3 event[3]: 0x80500 str=(0,2) 'do'
* pos=6 event[4]: 0x900500 str=(4,21) 'a deer, a female deer'
* pos=9 event[5]: 0x880500 str=(26,2) 're'
* pos=12 event[6]: 0x900500 str=(30,20) 'a drop of golden sun'
* pos=15 event[7]: 0x880500 str=(51,2) 'mi'
* pos=18 event[8]: 0x900500 str=(55,20) 'a name I call myself'
* pos=21 event[9]: 0x880500 str=(76,2) 'fa'
* pos=24 event[10]: 0x900500 str=(80,22) 'a long long way to run'
* pos=27 event[11]: 0x800020
* pos=28 event[12]: 0x8
* pos=29 event[13]: 0x2
*/
// buffer to where we will write the events
constexpr const int events_size = 100;
int events[events_size] = {};
static_assert(events_size >= sizeof(expected_events)/sizeof(expected_events[0]), "buffer too small");
// buffer for placing any scalars/tags that cannot be filtered
// in-place
char arena[100] = {};
// ensure the estimation will succeed vs required size
int estimated_size = c4::yml::extra::estimate_events_ints_size(yaml);
if (estimated_size > events_size)
{
printf("the estimated size (%d) will not fit the events array (%d)\n", estimated_size, events_size); // LCOV_EXCL_LINE
return 1; // LCOV_EXCL_LINE
}
// parse now. the parse should succeed (because the YAML above is
// legit), but if there were would be a parse error, we would get
// the default behavior which is abort on error, since we did not
// set up the error callbacks
c4::yml::extra::EventHandlerInts handler;
c4::yml::ParseEngine<c4::yml::extra::EventHandlerInts> parser(&handler);
handler.reset(yaml, arena, events, estimated_size); // note we pass the estimated size!
parser.parse_in_place_ev("filename", yaml);
// the YAML was successfully parsed, but it may happen that it
// requires more events than may fit in the buffer. so we need to
// check that it actually fits (this is mandatory):
if(!handler.fits_buffers())
{
printf("error: buffers too small: required_evt=%d actual_evt=%d\n required_arena=%zu actual_arena=%zu\n", // LCOV_EXCL_LINE
handler.required_size_events(), estimated_size, handler.required_size_arena(), c4::to_csubstr(arena).len); // LCOV_EXCL_LINE
// WATCHOUT: if you want to retry the parse, you need to set
// up the source buffer again, because it is invalidated from
// being parsed in place. refer to the doxygen documentation
// for more details.
return 1; // LCOV_EXCL_LINE
}
// done!
printf("success! YAML requires event size %d, estimated=%d (required_arena=%zu actual=%zu)\n", // LCOV_EXCL_LINE
handler.required_size_events(), estimated_size, handler.required_size_arena(), c4::to_csubstr(arena).len); // LCOV_EXCL_LINE
// ensure the result is as expected
bool compare = true;
// example iterating through the events array: compare and print
// the result
for (int pos = 0, evt = 0; pos < handler.required_size_events(); ++pos, ++evt)
{
bool status = (events[pos] == expected_events[pos]);
printf("pos=%d\tevent[%d]:\t0x%x", pos, evt, events[pos]);
if(events[pos] & WSTR) // the event has a string following it
{
int offset = events[pos + 1];
int length = events[pos + 2];
bool in_arena = (events[pos] & AREN);
// WATCHOUT! the string is NOT ZERO TERMINATED!
const char *ptr = in_arena ? arena : yaml;
const char *str = ptr + offset;
printf("\tstr=(%d,%d)\t'%.*s'", offset, length, length, str);
status = status && (offset == expected_events[pos + 1]);
status = status && (length == expected_events[pos + 2]);
pos += 2; // advance the two ints from the string
}
if(!status)
{
printf(" ... fail!"); // LCOV_EXCL_LINE
compare = false; // LCOV_EXCL_LINE
}
printf("\n");
}
return compare ? 0 : 1;
}
// NOLINTEND(hicpp-signed-bitwise)

49
samples/quickstart.cpp
View File

@@ -129,6 +129,9 @@ C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
C4_SUPPRESS_WARNING_GCC_CLANG("-Wcast-qual")
C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast")
C4_SUPPRESS_WARNING_GCC("-Wuseless-cast")
#if __GNUC__ >= 6
C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wnull-dereference")
#endif
namespace sample {
@@ -880,7 +883,7 @@ I am something: indeed
// ... and use it when querying
ryml::ConstNodeRef subject_node = tree2["bar"][1];
CHECK(subject_node.val() == "30");
ryml::Location loc = parser.location(subject_node);
ryml::Location loc = subject_node.location(parser);
CHECK(parser.location_contents(loc).begins_with("30"));
CHECK(loc.line == 1u);
CHECK(loc.col == 9u);
@@ -5284,7 +5287,7 @@ foo: [one, [two, three]]
// Now the structure will be built during parsing:
ryml::Tree tree = parse_in_arena(&parser, "source.yml", yaml);
// After this, we are ready to query the location from the parser:
ryml::Location loc = parser.location(tree.rootref());
ryml::Location loc = tree.rootref().location(parser);
// As for the complexity of the query: for large buffers it is
// O(log(numlines)). For short source buffers (30 lines and less),
// it is O(numlines), as a plain linear search is faster in this
@@ -5295,31 +5298,31 @@ foo: [one, [two, three]]
CHECK(loc.col == 0u);
// on the next call, we only pay O(log(numlines)) because the
// rebuild is already available:
loc = parser.location(tree["aa"]);
loc = tree["aa"].location(parser);
CHECK(parser.location_contents(loc).begins_with("aa"));
CHECK(loc.offset == 2u);
CHECK(loc.line == 1u);
CHECK(loc.col == 0u);
// KEYSEQ in flow style: points at the key
loc = parser.location(tree["foo"]);
loc = tree["foo"].location(parser);
CHECK(parser.location_contents(loc).begins_with("foo"));
CHECK(loc.offset == 16u);
CHECK(loc.line == 2u);
CHECK(loc.col == 0u);
loc = parser.location(tree["foo"][0]);
loc = tree["foo"][0].location(parser);
CHECK(parser.location_contents(loc).begins_with("one"));
CHECK(loc.line == 2u);
CHECK(loc.col == 6u);
// SEQ in flow style: location points at the opening '[' (there's no key)
loc = parser.location(tree["foo"][1]);
loc = tree["foo"][1].location(parser);
CHECK(parser.location_contents(loc).begins_with("["));
CHECK(loc.line == 2u);
CHECK(loc.col == 11u);
loc = parser.location(tree["foo"][1][0]);
loc = tree["foo"][1][0].location(parser);
CHECK(parser.location_contents(loc).begins_with("two"));
CHECK(loc.line == 2u);
CHECK(loc.col == 12u);
loc = parser.location(tree["foo"][1][1]);
loc = tree["foo"][1][1].location(parser);
CHECK(parser.location_contents(loc).begins_with("three"));
CHECK(loc.line == 2u);
CHECK(loc.col == 17u);
@@ -5332,8 +5335,8 @@ foo: [one, [two, three]]
ryml::Tree docval = parse_in_arena(&parser, "docval.yaml", "this is a docval");
// From now on, none of the locations from the previous tree can
// be queried:
//loc = parser.location(tree.rootref()); // ERROR, undefined behavior
loc = parser.location(docval.rootref()); // OK. this is the latest tree from this parser
//loc = tree.rootref().location(parser); // ERROR, undefined behavior
loc = docval.rootref().location(parser); // OK. this is the latest tree from this parser
CHECK(parser.location_contents(loc).begins_with("this is a docval"));
CHECK(loc.line == 0u);
CHECK(loc.col == 0u);
@@ -5361,63 +5364,63 @@ seq with key:
// points at the first child's key. For example, in this case
// the root does not have a key, so its location is taken
// to be at the first child:
loc = parser.location(tree2.rootref());
loc = tree2.rootref().location(parser);
CHECK(parser.location_contents(loc).begins_with("a new"));
CHECK(loc.offset == 1u);
CHECK(loc.line == 1u);
CHECK(loc.col == 0u);
// note the first child points exactly at the same place:
loc = parser.location(tree2["a new"]);
loc = tree2["a new"].location(parser);
CHECK(parser.location_contents(loc).begins_with("a new"));
CHECK(loc.offset == 1u);
CHECK(loc.line == 1u);
CHECK(loc.col == 0u);
loc = parser.location(tree2["to"]);
loc = tree2["to"].location(parser);
CHECK(parser.location_contents(loc).begins_with("to"));
CHECK(loc.line == 2u);
CHECK(loc.col == 0u);
// but of course, if the block-style map is a KEYMAP, then the
// location is the map's key, and not the first child's key:
loc = parser.location(tree2["map with key"]);
loc = tree2["map with key"].location(parser);
CHECK(parser.location_contents(loc).begins_with("map with key"));
CHECK(loc.line == 3u);
CHECK(loc.col == 0u);
loc = parser.location(tree2["map with key"]["first"]);
loc = tree2["map with key"]["first"].location(parser);
CHECK(parser.location_contents(loc).begins_with("first"));
CHECK(loc.line == 4u);
CHECK(loc.col == 2u);
loc = parser.location(tree2["map with key"]["second"]);
loc = tree2["map with key"]["second"].location(parser);
CHECK(parser.location_contents(loc).begins_with("second"));
CHECK(loc.line == 5u);
CHECK(loc.col == 2u);
// same thing for KEYSEQ:
loc = parser.location(tree2["seq with key"]);
loc = tree2["seq with key"].location(parser);
CHECK(parser.location_contents(loc).begins_with("seq with key"));
CHECK(loc.line == 6u);
CHECK(loc.col == 0u);
loc = parser.location(tree2["seq with key"][0]);
loc = tree2["seq with key"][0].location(parser);
CHECK(parser.location_contents(loc).begins_with("first value"));
CHECK(loc.line == 7u);
CHECK(loc.col == 4u);
loc = parser.location(tree2["seq with key"][1]);
loc = tree2["seq with key"][1].location(parser);
CHECK(parser.location_contents(loc).begins_with("second value"));
CHECK(loc.line == 8u);
CHECK(loc.col == 4u);
// SEQ nested in SEQ: container location points at the first child's "- " dash
loc = parser.location(tree2["seq with key"][2]);
loc = tree2["seq with key"][2].location(parser);
CHECK(parser.location_contents(loc).begins_with("- nested first value"));
CHECK(loc.line == 10u);
CHECK(loc.col == 4u);
loc = parser.location(tree2["seq with key"][2][0]);
loc = tree2["seq with key"][2][0].location(parser);
CHECK(parser.location_contents(loc).begins_with("nested first value"));
CHECK(loc.line == 10u);
CHECK(loc.col == 6u);
// MAP nested in SEQ: same as above: point to key
loc = parser.location(tree2["seq with key"][3]);
loc = tree2["seq with key"][3].location(parser);
CHECK(parser.location_contents(loc).begins_with("nested first: "));
CHECK(loc.line == 13u);
CHECK(loc.col == 4u);
loc = parser.location(tree2["seq with key"][3][0]);
loc = tree2["seq with key"][3][0].location(parser);
CHECK(parser.location_contents(loc).begins_with("nested first: "));
CHECK(loc.line == 13u);
CHECK(loc.col == 4u);

17
samples/singleheader-ints/CMakeLists.txt Normal file
View File

@@ -0,0 +1,17 @@
cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
project(ryml-quickstart-ints LANGUAGES CXX)
# create a target to amalgamate ryml into a single header:
include(amalgamate.cmake)
amalgamate_ryml(SINGLE_HEADER_DIR SINGLE_HEADER)
# now simply define the executable:
add_executable(ryml-quickstart-ints ../quickstart-ints.cpp ${SINGLE_HEADER})
target_compile_features(ryml-quickstart-ints PUBLIC cxx_std_11)
target_compile_definitions(ryml-quickstart-ints PUBLIC -DRYML_SINGLE_HEADER)
target_compile_definitions(ryml-quickstart-ints PUBLIC -DRYML_SINGLE_HEADER_INTS)
target_include_directories(ryml-quickstart-ints PUBLIC "${SINGLE_HEADER_DIR}")
add_custom_target(run ryml-quickstart-ints
COMMAND $<TARGET_FILE:ryml-quickstart-ints>
DEPENDS ryml-quickstart-ints)

18
samples/singleheader-ints/amalgamate.cmake Normal file
View File

@@ -0,0 +1,18 @@
find_package(Python3 COMPONENTS Interpreter)
# amalgamate ryml to get the single header
function(amalgamate_ryml header_dir header_file)
set(rymldir "${CMAKE_CURRENT_LIST_DIR}/../..")
set(singleheaderdir "${rymldir}/src_singleheader")
set(singleheader "${singleheaderdir}/ryml_ints.hpp")
set(amscript "${rymldir}/tools/amalgamate.py")
file(GLOB_RECURSE srcfiles
LIST_DIRECTORIES FALSE
CONFIGURE_DEPENDS "${rymldir}/src" "${rymldir}/src_extra")
add_custom_command(OUTPUT "${singleheader}"
COMMAND "${Python3_EXECUTABLE}" "${amscript}" -e ints -- "${singleheader}"
COMMENT "${Python3_EXECUTABLE} ${amscript} -e ints -- ${singleheader}"
DEPENDS ${srcfiles} "${amscript}" "${rymldir}/ext/c4core/cmake/amalgamate_utils.py")
set(${header_dir} "${singleheaderdir}" PARENT_SCOPE)
set(${header_file} "${singleheader}" PARENT_SCOPE)
endfunction()

10
samples/singleheader-ints/run.sh Executable file
View File

@@ -0,0 +1,10 @@
#!/bin/bash -x
# take the build type from the command line, or default to release
cfg=${1:-Release}
# make sure to run from where this file is
cd $(dirname $0)
# configure the sample
cmake -S . -B ./build/$cfg -DCMAKE_BUILD_TYPE=$cfg
# build and run the sample
cmake --build ./build/$cfg --config $cfg --target run

27
samples/singleheaderlib-ints/CMakeLists.txt Normal file
View File

@@ -0,0 +1,27 @@
cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
project(ryml-quickstart-ints LANGUAGES CXX)
# create a target to amalgamate ryml into a single header:
include(../singleheader-ints/amalgamate.cmake)
amalgamate_ryml(SINGLE_HEADER_DIR SINGLE_HEADER)
# add a library using the amalgamated header
add_library(ryml lib.cpp ${SINGLE_HEADER})
target_compile_features(ryml PUBLIC cxx_std_11)
target_include_directories(ryml PUBLIC "${SINGLE_HEADER_DIR}")
# now simply define the executable:
add_executable(ryml-quickstart-ints ../quickstart-ints.cpp)
target_link_libraries(ryml-quickstart-ints PRIVATE ryml)
target_compile_definitions(ryml-quickstart-ints PUBLIC -DRYML_SINGLE_HEADER_LIB)
target_compile_definitions(ryml-quickstart-ints PUBLIC -DRYML_SINGLE_HEADER_INTS)
# adjustments for shared library
if(BUILD_SHARED_LIBS)
# RYML_SHARED should be propagated to targets consuming ryml
target_compile_definitions(ryml PUBLIC -DRYML_SHARED)
endif()
add_custom_target(run ryml-quickstart-ints
COMMAND $<TARGET_FILE:ryml-quickstart-ints>
DEPENDS ryml-quickstart-ints)

2
samples/singleheaderlib-ints/lib.cpp Normal file
View File

@@ -0,0 +1,2 @@
#define RYML_SINGLE_HDR_DEFINE_NOW
#include <ryml_ints.hpp>

10
samples/singleheaderlib-ints/run_shared.sh Executable file
View File

@@ -0,0 +1,10 @@
#!/bin/bash -x
# take the build type from the command line, or default to release
cfg=${1:-Release}
# make sure to run from where this file is
cd $(dirname $0)
# configure the sample
cmake -S . -B ./build/$cfg-shared -DCMAKE_BUILD_TYPE=$cfg -DBUILD_SHARED_LIBS=ON
# build and run the sample
cmake --build ./build/$cfg-shared --config $cfg --target run

10
samples/singleheaderlib-ints/run_static.sh Executable file
View File

@@ -0,0 +1,10 @@
#!/bin/bash -x
# take the build type from the command line, or default to release
cfg=${1:-Release}
# make sure to run from where this file is
cd $(dirname $0)
# configure the sample
cmake -S . -B ./build/$cfg-static -DCMAKE_BUILD_TYPE=$cfg -DBUILD_SHARED_LIBS=OFF
# build and run the sample
cmake --build ./build/$cfg-static --config $cfg --target run

36
src/c4/yml/detail/parser_dbg.hpp → src/c4/yml/detail/dbgprint.hpp
View File

@@ -1,5 +1,5 @@
#ifndef _C4_YML_DETAIL_PARSER_DBG_HPP_
#define _C4_YML_DETAIL_PARSER_DBG_HPP_
#ifndef _C4_YML_DETAIL_DBGPRINT_HPP_
#define _C4_YML_DETAIL_DBGPRINT_HPP_
#ifndef _C4_YML_COMMON_HPP_
#include "../common.hpp"
@@ -11,22 +11,9 @@
//-----------------------------------------------------------------------------
// some debugging scaffolds
// NOLINTBEGIN(*)
C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
C4_SUPPRESS_WARNING_MSVC_PUSH
C4_SUPPRESS_WARNING_MSVC(4068/*unknown pragma*/)
#pragma GCC system_header
C4_SUPPRESS_WARNING_GCC("-Wunknown-pragmas")
C4_SUPPRESS_WARNING_CLANG("-Wgnu-zero-variadic-macro-arguments")
// NOLINTEND(*)
// debug prints
#ifndef RYML_DBG
# define _c4err(fmt, ...) \
this->_err("ERROR: " fmt, ## __VA_ARGS__)
# define _c4dbgt(fmt, ...)
# define _c4dbgpf(fmt, ...)
# define _c4dbgpf_(fmt, ...)
@@ -36,14 +23,12 @@ C4_SUPPRESS_WARNING_CLANG("-Wgnu-zero-variadic-macro-arguments")
# define _c4presc(...)
# define _c4prscalar(msg, scalar, keep_newlines)
#else
# define _c4err(fmt, ...) \
do { RYML_DEBUG_BREAK(); this->_err("ERROR:\n" "{}:{}: " fmt, __FILE__, __LINE__, ## __VA_ARGS__); } while(0)
# define _c4dbgt(fmt, ...) do { if(_dbg_enabled()) { \
this->_dbg ("{}:{}: " fmt , __FILE__, __LINE__, ## __VA_ARGS__); } } while(0)
# define _c4dbgpf(fmt, ...) _dbg_printf("{}:{}: " fmt "\n", __FILE__, __LINE__, ## __VA_ARGS__)
# define _c4dbgpf_(fmt, ...) _dbg_printf("{}:{}: " fmt , __FILE__, __LINE__, ## __VA_ARGS__)
# define _c4dbgp(msg) _dbg_printf("{}:{}: " msg "\n", __FILE__, __LINE__ )
# define _c4dbgp_(msg) _dbg_printf("{}:{}: " msg , __FILE__, __LINE__ )
this->_dbg ("{}:{}: " fmt , __FILE__, __LINE__, __VA_ARGS__); } } while(0)
# define _c4dbgpf(fmt, ...) _dbg_printf("{}:{}: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
# define _c4dbgpf_(fmt, ...) _dbg_printf("{}:{}: " fmt , __FILE__, __LINE__, __VA_ARGS__)
# define _c4dbgp(msg) _dbg_printf("{}:{}: " msg "\n", __FILE__, __LINE__ )
# define _c4dbgp_(msg) _dbg_printf("{}:{}: " msg , __FILE__, __LINE__ )
# define _c4dbgq(msg) _dbg_printf(msg "\n")
# define _c4presc(...) do { if(_dbg_enabled()) __c4presc(__VA_ARGS__); } while(0)
# define _c4prscalar(msg, scalar, keep_newlines) \
@@ -141,7 +126,4 @@ inline void __c4presc(csubstr s, bool keep_newlines=false)
#endif // RYML_DBG
C4_SUPPRESS_WARNING_GCC_CLANG_POP
C4_SUPPRESS_WARNING_MSVC_POP
#endif /* _C4_YML_DETAIL_PARSER_DBG_HPP_ */
#endif /* _C4_YML_DETAIL_DBGPRINT_HPP_ */

4
src/c4/yml/emit.def.hpp
View File

@@ -6,8 +6,8 @@
#endif
/** @file emit.def.hpp Definitions for emit functions. */
#ifndef _C4_YML_DETAIL_PARSER_DBG_HPP_
#include "c4/yml/detail/parser_dbg.hpp"
#ifndef _C4_YML_DETAIL_DBGPRINT_HPP_
#include "c4/yml/detail/dbgprint.hpp"
#endif
namespace c4 {

4
src/c4/yml/event_handler_stack.hpp
View File

@@ -9,8 +9,8 @@
#include "c4/yml/node_type.hpp"
#endif
#ifndef _C4_YML_DETAIL_PARSER_DBG_HPP_
#include "c4/yml/detail/parser_dbg.hpp"
#ifndef _C4_YML_DETAIL_DBGPRINT_HPP_
#include "c4/yml/detail/dbgprint.hpp"
#endif
#ifndef _C4_YML_PARSER_STATE_HPP_

13
src/c4/yml/event_handler_tree.hpp
View File

@@ -19,11 +19,12 @@ namespace yml {
* @{ */
/** The stack state needed specifically by @ref EventHandlerTree */
/** @cond dev */
struct EventHandlerTreeState : public ParserState
{
NodeData *tr_data;
};
/** @endcond */
/** The event handler to create a ryml @ref Tree. See the
@@ -463,9 +464,8 @@ public:
{
_c4dbgpf("node[{}]: set key anchor: [{}]~~~{}~~~", m_curr->node_id, anchor.len, anchor);
_RYML_CB_ASSERT(m_stack.m_callbacks, m_tree);
if(C4_UNLIKELY(_has_any_(KEYREF)))
_RYML_CB_ERR_(m_tree->callbacks(), "key cannot have both anchor and ref", m_curr->pos);
_RYML_CB_ASSERT(m_tree->callbacks(), !anchor.begins_with('&'));
_RYML_CB_ASSERT(m_stack.m_callbacks, !_has_any_(KEYREF));
_RYML_CB_ASSERT(m_stack.m_callbacks, !anchor.begins_with('&'));
_enable_(KEYANCH);
m_curr->tr_data->m_key.anchor = anchor;
}
@@ -473,9 +473,8 @@ public:
{
_c4dbgpf("node[{}]: set val anchor: [{}]~~~{}~~~", m_curr->node_id, anchor.len, anchor);
_RYML_CB_ASSERT(m_stack.m_callbacks, m_tree);
if(C4_UNLIKELY(_has_any_(VALREF)))
_RYML_CB_ERR_(m_tree->callbacks(), "val cannot have both anchor and ref", m_curr->pos);
_RYML_CB_ASSERT(m_tree->callbacks(), !anchor.begins_with('&'));
_RYML_CB_ASSERT(m_stack.m_callbacks, !_has_any_(VALREF));
_RYML_CB_ASSERT(m_stack.m_callbacks, !anchor.begins_with('&'));
_enable_(VALANCH);
m_curr->tr_data->m_val.anchor = anchor;
}

2
src/c4/yml/filter_processor.hpp
View File

@@ -5,7 +5,7 @@
#ifdef RYML_DBG
#include "c4/charconv.hpp"
#include "c4/yml/detail/parser_dbg.hpp"
#include "c4/yml/detail/dbgprint.hpp"
#endif
namespace c4 {

14
src/c4/yml/node.hpp
View File

@@ -611,6 +611,19 @@ public:
/** @} */
public:
/** @name locations */
/** @{ */
Location location(Parser const& parser) const
{
_C4RR();
return tree_->location(parser, id_);
}
/** @} */
public:
/** @name deserialization */
@@ -884,7 +897,6 @@ public:
ConstNodeRef& operator= (NodeRef const&) noexcept;
ConstNodeRef& operator= (NodeRef &&) noexcept;
/** @} */
public:

3
src/c4/yml/node_type.hpp
View File

@@ -8,6 +8,9 @@
C4_SUPPRESS_WARNING_MSVC_PUSH
C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast")
#if __GNUC__ >= 6
C4_SUPPRESS_WARNING_GCC("-Wnull-dereference")
#endif
namespace c4 {
namespace yml {

18
src/c4/yml/parse.cpp
View File

@@ -129,24 +129,6 @@ Tree parse_json_in_arena(csubstr filename, csubstr json
Tree parse_json_in_arena( csubstr json ) { Parser::handler_type event_handler; Parser parser(&event_handler); Tree tree(parser.callbacks()); substr src = tree.copy_to_arena(json); parse_json_in_place(&parser, {} , src, &tree, tree.root_id()); return tree; }
RYML_EXPORT C4_NO_INLINE size_t _find_last_newline_and_larger_indentation(csubstr s, size_t indentation) noexcept
{
if(indentation + 1 > s.len)
return npos;
for(size_t i = s.len-indentation-1; i != size_t(-1); --i)
{
if(s.str[i] == '\n')
{
csubstr rem = s.sub(i + 1);
size_t first = rem.first_not_of(' ');
first = (first != npos) ? first : rem.len;
if(first > indentation)
return i;
}
}
return npos;
}
//-----------------------------------------------------------------------------
RYML_EXPORT id_type estimate_tree_capacity(csubstr src)

186
src/c4/yml/parse_engine.def.hpp
View File

@@ -8,11 +8,16 @@
#include <ctype.h>
#include "c4/yml/detail/parser_dbg.hpp"
#include "c4/yml/detail/dbgprint.hpp"
#include "c4/yml/filter_processor.hpp"
#ifdef RYML_DBG
#include <c4/dump.hpp>
#include "c4/yml/detail/print.hpp"
#define _c4err_(fmt, ...) do { RYML_DEBUG_BREAK(); this->_err("ERROR:\n" "{}:{}: " fmt, __FILE__, __LINE__, __VA_ARGS__); } while(0)
#define _c4err(fmt) do { RYML_DEBUG_BREAK(); this->_err("ERROR:\n" "{}:{}: " fmt, __FILE__, __LINE__); } while(0)
#else
#define _c4err_(fmt, ...) this->_err("ERROR: " fmt, __VA_ARGS__)
#define _c4err(fmt) this->_err("ERROR: {}", fmt)
#endif
@@ -796,7 +801,7 @@ bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow(csubstr s)
case '{':
case '[':
//_RYML_WITHOUT_TAB_TOKENS(case '\t'):
_c4err("invalid token \":{}\"", _c4prc(s.str[1]));
_c4err_("invalid token \":{}\"", _c4prc(s.str[1]));
break;
case ' ':
case '}':
@@ -831,7 +836,7 @@ bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow(csubstr s)
case '}':
case '[':
case ']':
_c4err("invalid token \"?{}\"", _c4prc(s.str[1]));
_c4err_("invalid token \"?{}\"", _c4prc(s.str[1]));
break;
default:
break;
@@ -945,7 +950,7 @@ bool ParseEngine<EventHandler>::_scan_scalar_plain_seq_flow(ScannedScalar *C4_RE
case '{':
case '}':
_line_progressed(i);
_c4err("invalid character: '{}'", c); // noreturn
_c4err_("invalid character: '{}'", c); // noreturn
default:
;
}
@@ -1021,14 +1026,14 @@ bool ParseEngine<EventHandler>::_scan_scalar_plain_map_flow(ScannedScalar *C4_RE
case '{':
case '[':
_line_progressed(i);
_c4err("invalid character: '{}'", c); // noreturn
_c4err_("invalid character: '{}'", c); // noreturn
break;
case ']':
_line_progressed(i);
if(has_any(RSEQIMAP))
goto ended_scalar;
else
_c4err("invalid character: '{}'", c); // noreturn
_c4err_("invalid character: '{}'", c); // noreturn
break;
case '#':
if(!i || s.str[i-1] == ' ' _RYML_WITH_TAB_TOKENS(|| s.str[i-1] == '\t'))
@@ -2664,16 +2669,16 @@ void ParseEngine<EventHandler>::_filter_dquoted_backslash(FilterProcessor &C4_RE
else if(next == 'x') // 2-digit Unicode escape (\xXX), code point 0x00–0xFF
{
if(C4_UNLIKELY(proc.rpos + 1u + 2u >= proc.src.len))
_c4err("\\x requires 2 hex digits. scalar pos={}", proc.rpos);
_c4err_("\\x requires 2 hex digits. scalar pos={}", proc.rpos);
char readbuf[8];
csubstr codepoint = proc.src.sub(proc.rpos + 2u, 2u);
_c4dbgfdq("utf8 ~~~{}~~~ rpos={} rem=~~~{}~~~", codepoint, proc.rpos, proc.src.sub(proc.rpos));
uint32_t codepoint_val = {};
if(C4_UNLIKELY(!read_hex(codepoint, &codepoint_val)))
_c4err("failed to read \\x codepoint. scalar pos={}", proc.rpos);
_c4err_("failed to read \\x codepoint. scalar pos={}", proc.rpos);
const size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val);
if(C4_UNLIKELY(numbytes == 0))
_c4err("failed to decode code point={}", proc.rpos);
_c4err_("failed to decode code point={}", proc.rpos);
_RYML_CB_ASSERT(callbacks(), numbytes <= 4);
proc.translate_esc_bulk(readbuf, numbytes, /*nread*/3u);
_c4dbgfdq("utf8 after rpos={} rem=~~~{}~~~", proc.rpos, proc.src.sub(proc.rpos));
@@ -2681,30 +2686,30 @@ void ParseEngine<EventHandler>::_filter_dquoted_backslash(FilterProcessor &C4_RE
else if(next == 'u') // 4-digit Unicode escape (\uXXXX), code point 0x0000–0xFFFF
{
if(C4_UNLIKELY(proc.rpos + 1u + 4u >= proc.src.len))
_c4err("\\u requires 4 hex digits. scalar pos={}", proc.rpos);
_c4err_("\\u requires 4 hex digits. scalar pos={}", proc.rpos);
char readbuf[8];
csubstr codepoint = proc.src.sub(proc.rpos + 2u, 4u);
uint32_t codepoint_val = {};
if(C4_UNLIKELY(!read_hex(codepoint, &codepoint_val)))
_c4err("failed to parse \\u codepoint. scalar pos={}", proc.rpos);
_c4err_("failed to parse \\u codepoint. scalar pos={}", proc.rpos);
const size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val);
if(C4_UNLIKELY(numbytes == 0))
_c4err("failed to decode code point={}", proc.rpos);
_c4err_("failed to decode code point={}", proc.rpos);
_RYML_CB_ASSERT(callbacks(), numbytes <= 4);
proc.translate_esc_bulk(readbuf, numbytes, /*nread*/5u);
}
else if(next == 'U') // 8-digit Unicode escape (\UXXXXXXXX), full 32-bit code point
{
if(C4_UNLIKELY(proc.rpos + 1u + 8u >= proc.src.len))
_c4err("\\U requires 8 hex digits. scalar pos={}", proc.rpos);
_c4err_("\\U requires 8 hex digits. scalar pos={}", proc.rpos);
char readbuf[8];
csubstr codepoint = proc.src.sub(proc.rpos + 2u, 8u);
uint32_t codepoint_val = {};
if(C4_UNLIKELY(!read_hex(codepoint, &codepoint_val)))
_c4err("failed to parse \\U codepoint. scalar pos={}", proc.rpos);
_c4err_("failed to parse \\U codepoint. scalar pos={}", proc.rpos);
const size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val);
if(C4_UNLIKELY(numbytes == 0))
_c4err("failed to decode code point={}", proc.rpos);
_c4err_("failed to decode code point={}", proc.rpos);
_RYML_CB_ASSERT(callbacks(), numbytes <= 4);
proc.translate_esc_bulk(readbuf, numbytes, /*nread*/9u);
}
@@ -2777,7 +2782,7 @@ void ParseEngine<EventHandler>::_filter_dquoted_backslash(FilterProcessor &C4_RE
}
else
{
_c4err("unknown character '{}' after '\\' pos={}", _c4prc(next), proc.rpos);
_c4err_("unknown character '{}' after '\\' pos={}", _c4prc(next), proc.rpos);
}
_c4dbgfdq("backslash...sofar=[{}]~~~{}~~~", proc.wpos, proc.sofar());
}
@@ -2854,6 +2859,24 @@ FilterResultExtending ParseEngine<EventHandler>::filter_scalar_dquoted_in_place(
//-----------------------------------------------------------------------------
// block filtering helpers
C4_NO_INLINE inline size_t _find_last_newline_and_larger_indentation(csubstr s, size_t indentation) noexcept
{
if(indentation + 1 > s.len)
return npos;
for(size_t i = s.len-indentation-1; i != size_t(-1); --i)
{
if(s.str[i] == '\n')
{
csubstr rem = s.sub(i + 1);
size_t first = rem.first_not_of(' ');
first = (first != npos) ? first : rem.len;
if(first > indentation)
return i;
}
}
return npos;
}
template<class EventHandler>
template<class FilterProcessor>
void ParseEngine<EventHandler>::_filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp, size_t indentation)
@@ -3511,13 +3534,17 @@ csubstr ParseEngine<EventHandler>::_filter_scalar_dquot(substr s)
_c4dbgpf("filtering dquo scalar: not enough space: needs {}, have {}", len, s.len);
substr dst = m_evt_handler->alloc_arena(len, &s);
_c4dbgpf("filtering dquo scalar: dst.len={}", dst.len);
_RYML_CB_ASSERT(this->callbacks(), dst.len == len);
FilterResult rsd = this->filter_scalar_dquoted(s, dst);
_c4dbgpf("filtering dquo scalar: ... result now needs {} was {}", rsd.required_len(), len);
_RYML_CB_ASSERT(this->callbacks(), rsd.required_len() <= len); // may be smaller!
_RYML_CB_CHECK(m_evt_handler->m_stack.m_callbacks, rsd.valid());
_c4dbgpf("filtering dquo scalar: success! s=[{}]~~~{}~~~", rsd.get().len, rsd.get());
return rsd.get();
if(dst.str)
{
_RYML_CB_ASSERT(this->callbacks(), dst.len == len);
FilterResult rsd = this->filter_scalar_dquoted(s, dst);
_c4dbgpf("filtering dquo scalar: ... result now needs {} was {}", rsd.required_len(), len);
_RYML_CB_ASSERT(this->callbacks(), rsd.required_len() <= len); // may be smaller!
_RYML_CB_CHECK(m_evt_handler->m_stack.m_callbacks, rsd.valid());
_c4dbgpf("filtering dquo scalar: success! s=[{}]~~~{}~~~", rsd.get().len, rsd.get());
return rsd.get();
}
return dst;
}
}
@@ -3881,119 +3908,6 @@ csubstr ParseEngine<EventHandler>::location_contents(Location const& loc) const
return m_buf.sub(loc.offset);
}
template<class EventHandler>
Location ParseEngine<EventHandler>::location(ConstNodeRef node) const
{
_RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks, node.readable());
return location(*node.tree(), node.id());
}
template<class EventHandler>
Location ParseEngine<EventHandler>::location(Tree const& tree, id_type node) const
{
// try hard to avoid getting the location from a null string.
Location loc;
if(_location_from_node(tree, node, &loc, 0))
return loc;
return val_location(m_buf.str);
}
template<class EventHandler>
bool ParseEngine<EventHandler>::_location_from_node(Tree const& tree, id_type node, Location *C4_RESTRICT loc, id_type level) const
{
if(tree.has_key(node))
{
csubstr k = tree.key(node);
if(C4_LIKELY(k.str != nullptr))
{
_RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks, k.is_sub(m_buf));
_RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks, m_buf.is_super(k));
*loc = val_location(k.str);
return true;
}
}
if(tree.has_val(node))
{
csubstr v = tree.val(node);
if(C4_LIKELY(v.str != nullptr))
{
_RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks, v.is_sub(m_buf));
_RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks, m_buf.is_super(v));
*loc = val_location(v.str);
return true;
}
}
if(tree.is_container(node))
{
if(_location_from_cont(tree, node, loc))
return true;
}
if(tree.type(node) != NOTYPE && level == 0)
{
// try the prev sibling
{
const id_type prev = tree.prev_sibling(node);
if(prev != NONE)
{
if(_location_from_node(tree, prev, loc, level+1))
return true;
}
}
// try the next sibling
{
const id_type next = tree.next_sibling(node);
if(next != NONE)
{
if(_location_from_node(tree, next, loc, level+1))
return true;
}
}
// try the parent
{
const id_type parent = tree.parent(node);
if(parent != NONE)
{
if(_location_from_node(tree, parent, loc, level+1))
return true;
}
}
}
return false;
}
template<class EventHandler>
bool ParseEngine<EventHandler>::_location_from_cont(Tree const& tree, id_type node, Location *C4_RESTRICT loc) const
{
_RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks, tree.is_container(node));
if(!tree.is_stream(node))
{
const char *node_start = tree._p(node)->m_val.scalar.str; // this was stored in the container
if(tree.has_children(node))
{
id_type child = tree.first_child(node);
if(tree.has_key(child))
{
// when a map starts, the container was set after the key
csubstr k = tree.key(child);
if(k.str && node_start > k.str)
node_start = k.str;
}
}
*loc = val_location(node_start);
return true;
}
else // it's a stream
{
*loc = val_location(m_buf.str); // just return the front of the buffer
}
return true;
}
template<class EventHandler>
Location ParseEngine<EventHandler>::val_location(const char *val) const
{

53
src/c4/yml/parse_engine.hpp
View File

@@ -1,10 +1,6 @@
#ifndef _C4_YML_PARSE_ENGINE_HPP_
#define _C4_YML_PARSE_ENGINE_HPP_
#ifndef _C4_YML_DETAIL_PARSER_DBG_HPP_
#include "c4/yml/detail/parser_dbg.hpp"
#endif
#ifndef _C4_YML_PARSER_STATE_HPP_
#include "c4/yml/parser_state.hpp"
#endif
@@ -41,7 +37,10 @@ namespace yml {
* - @ref EventHandlerTree is the handler responsible for creating the
* ryml @ref Tree
*
* - @ref EventHandlerYamlStd is the handler responsible for emitting
* - @ref extra::EventHandlerInts parses YAML into an integer array
representation of the tree and scalars.
*
* - @ref extra::EventHandlerTestSuite is the handler responsible for emitting
* standardized [YAML test suite
* events](https://github.com/yaml/yaml-test-suite), used (only) in
* the CI of this project.
@@ -103,8 +102,8 @@ namespace yml {
* cases. They are called by the parser when a just-handled
* value/container is actually the first key of a new map:
*
* - `actually_val_is_first_key_of_new_map_flow()` (@ref EventHandlerTree::actually_val_is_first_key_of_new_map_flow() "see implementation in EventHandlerTree" / @ref EventHandlerYamlStd::actually_val_is_first_key_of_new_map_flow() "see implementation in EventHandlerYamlStd")
* - `actually_val_is_first_key_of_new_map_block()` (@ref EventHandlerTree::actually_val_is_first_key_of_new_map_block() "see implementation in EventHandlerTree" / @ref EventHandlerYamlStd::actually_val_is_first_key_of_new_map_block() "see implementation in EventHandlerYamlStd")
* - `actually_val_is_first_key_of_new_map_flow()` (@ref EventHandlerTree::actually_val_is_first_key_of_new_map_flow() "see implementation in EventHandlerTree" / @ref EventHandlerTestSuite::actually_val_is_first_key_of_new_map_flow() "see implementation in EventHandlerTestSuite")
* - `actually_val_is_first_key_of_new_map_block()` (@ref EventHandlerTree::actually_val_is_first_key_of_new_map_block() "see implementation in EventHandlerTree" / @ref EventHandlerTestSuite::actually_val_is_first_key_of_new_map_block() "see implementation in EventHandlerTestSuite")
*
* For example, consider an implicit map inside a seq: `[a: b, c:
* d]` which is parsed as `[{a: b}, {c: d}]`. The standard event
@@ -282,11 +281,20 @@ public:
* - @ref EventHandlerTree is the handler responsible for creating the
* ryml @ref Tree
*
* - @ref EventHandlerYamlStd is the handler responsible for emitting
* - @ref extra::EventHandlerTestSuite is a handler responsible for emitting
* standardized [YAML test suite
* events](https://github.com/yaml/yaml-test-suite), used (only) in
* the CI of this project. This is not part of the library and is
* not installed.
*
* - @ref extra::EventHandlerInts is the handler responsible for
* emitting integer-coded events. It is intended for implementing
* fully-conformant parsing in other programming languages
* (integration is currently under work for
* [YamlScript](https://github.com/yaml/yamlscript) and
* [go-yaml](https://github.com/yaml/go-yaml/)). It is not part of
* the library and is not installed.
*
*/
template<class EventHandler>
class ParseEngine
@@ -393,8 +401,7 @@ public:
public:
/** @name deprecated parse methods
* @{ */
// deprecated parse methods
/** @cond dev */
template<class U=EventHandler> RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if<U::is_wtree, void>::type parse_in_place(csubstr filename, substr yaml, Tree *t, size_t node_id);
@@ -423,20 +430,15 @@ public:
template<class U=EventHandler> RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if<U::is_wtree, Tree>::type parse_in_arena( substr yaml );
/** @endcond */
/** @} */
public:
/** @name locations */
/** @{ */
/** Get the location of a node of the last tree to be parsed by this parser. */
Location location(Tree const& tree, id_type node_id) const;
/** Get the location of a node of the last tree to be parsed by this parser. */
Location location(ConstNodeRef node) const;
/** Get the string starting at a particular location, to the end
* of the parsed source buffer. */
csubstr location_contents(Location const& loc) const;
/** Given a pointer to a buffer position, get the location.
* @param[in] val must be pointing to somewhere in the source
* buffer that was last parsed by this object. */
@@ -444,6 +446,18 @@ public:
/** @} */
public:
/** @cond dev */
template<class U>
RYML_DEPRECATED("moved to Tree::location(Parser const&). deliberately undefined here.")
auto location(Tree const&, id_type node) const -> typename std::enable_if<U::is_wtree, Location>::type;
template<class U>
RYML_DEPRECATED("moved to ConstNodeRef::location(Parser const&), deliberately undefined here.")
auto location(ConstNodeRef const&) const -> typename std::enable_if<U::is_wtree, Location>::type;
/** @endcond */
public:
/** @name scalar filtering */
@@ -672,9 +686,6 @@ private:
void _resize_locations(size_t sz);
bool _locations_dirty() const;
bool _location_from_cont(Tree const& tree, id_type node, Location *C4_RESTRICT loc) const;
bool _location_from_node(Tree const& tree, id_type node, Location *C4_RESTRICT loc, id_type level) const;
private:
void _reset();
@@ -757,10 +768,6 @@ private:
};
/** @cond dev */
RYML_EXPORT C4_NO_INLINE size_t _find_last_newline_and_larger_indentation(csubstr s, size_t indentation) noexcept;
/** @endcond */
/** Quickly inspect the source to estimate the number of nodes the
* resulting tree is likely have. If a tree is empty before

2
src/c4/yml/preprocess.cpp
View File

@@ -1,5 +1,5 @@
#include "c4/yml/preprocess.hpp"
#include "c4/yml/detail/parser_dbg.hpp"
#include "c4/yml/detail/dbgprint.hpp"
/** @file preprocess.hpp Functions for preprocessing YAML prior to parsing. */

2
src/c4/yml/reference_resolver.cpp
View File

@@ -1,6 +1,6 @@
#include "c4/yml/reference_resolver.hpp"
#include "c4/yml/common.hpp"
#include "c4/yml/detail/parser_dbg.hpp"
#include "c4/yml/detail/dbgprint.hpp"
#ifdef RYML_DBG
#include "c4/yml/detail/print.hpp"
#else

49
src/c4/yml/tag.cpp
View File

@@ -1,6 +1,5 @@
#include "c4/yml/tag.hpp"
#include "c4/yml/tree.hpp"
#include "c4/yml/detail/parser_dbg.hpp"
#include "c4/yml/detail/dbgprint.hpp"
namespace c4 {
@@ -222,25 +221,7 @@ bool TagDirective::create_from_str(csubstr directive_)
return true;
}
bool TagDirective::create_from_str(csubstr directive_, Tree *tree)
{
_RYML_CB_CHECK(tree->callbacks(), directive_.begins_with("%TAG "));
if(!create_from_str(directive_))
{
_RYML_CB_ERR(tree->callbacks(), "invalid tag directive");
}
next_node_id = tree->size();
if(!tree->empty())
{
const id_type prev = tree->size() - 1;
if(tree->is_root(prev) && tree->type(prev) != NOTYPE && !tree->is_stream(prev))
++next_node_id;
}
_c4dbgpf("%TAG: handle={} prefix={} next_node={}", handle, prefix, next_node_id);
return true;
}
size_t TagDirective::transform(csubstr tag, substr output, Callbacks const& callbacks) const
size_t TagDirective::transform(csubstr tag, substr output, Callbacks const& callbacks, bool with_brackets) const
{
_c4dbgpf("%TAG: handle={} prefix={} next_node={}. tag={}", handle, prefix, next_node_id, tag);
_RYML_CB_ASSERT(callbacks, tag.len >= handle.len);
@@ -258,16 +239,26 @@ size_t TagDirective::transform(csubstr tag, substr output, Callbacks const& call
return 0; // return 0 to signal that the tag is local and cannot be resolved
}
}
size_t len = 1u + prefix.len + rest.len + 1u;
size_t len = prefix.len + rest.len;
if(with_brackets)
len += 2;
size_t numpc = rest.count('%');
if(numpc == 0)
{
if(len <= output.len)
{
output.str[0] = '<';
memcpy(1u + output.str, prefix.str, prefix.len);
memcpy(1u + output.str + prefix.len, rest.str, rest.len);
output.str[1u + prefix.len + rest.len] = '>';
if(with_brackets)
{
output.str[0] = '<';
memcpy(1u + output.str, prefix.str, prefix.len);
memcpy(1u + output.str + prefix.len, rest.str, rest.len);
output.str[1u + prefix.len + rest.len] = '>';
}
else
{
memcpy(output.str, prefix.str, prefix.len);
memcpy(output.str + prefix.len, rest.str, rest.len);
}
}
}
else
@@ -290,7 +281,8 @@ size_t TagDirective::transform(csubstr tag, substr output, Callbacks const& call
size_t prev = 0, wpos = 0;
auto appendstr = [&](csubstr s) { memcpy(output.str + wpos, s.str, s.len); wpos += s.len; };
auto appendchar = [&](char c) { output.str[wpos++] = c; };
appendchar('<');
if(with_brackets)
appendchar('<');
appendstr(prefix);
pos = rest.find('%');
_RYML_CB_ASSERT(callbacks, pos != npos);
@@ -312,7 +304,8 @@ size_t TagDirective::transform(csubstr tag, substr output, Callbacks const& call
_RYML_CB_ASSERT(callbacks, prev > 0);
_RYML_CB_ASSERT(callbacks, rest.len >= prev);
appendstr(rest.sub(prev));
appendchar('>');
if(with_brackets)
appendchar('>');
_RYML_CB_ASSERT(callbacks, wpos == len);
}
}

3
src/c4/yml/tag.hpp
View File

@@ -64,8 +64,7 @@ struct RYML_EXPORT TagDirective
id_type next_node_id;
bool create_from_str(csubstr directive_); ///< leaves next_node_id unfilled
bool create_from_str(csubstr directive_, Tree *tree);
size_t transform(csubstr tag, substr output, Callbacks const& callbacks) const;
size_t transform(csubstr tag, substr output, Callbacks const& callbacks, bool with_brackets=true) const;
};
struct RYML_EXPORT TagDirectiveRange

147
src/c4/yml/tree.cpp
View File

@@ -1,5 +1,5 @@
#include "c4/yml/tree.hpp"
#include "c4/yml/detail/parser_dbg.hpp"
#include "c4/yml/detail/dbgprint.hpp"
#include "c4/yml/node.hpp"
#include "c4/yml/reference_resolver.hpp"
@@ -857,7 +857,13 @@ void Tree::set_root_as_stream()
void Tree::remove_children(id_type node)
{
_RYML_CB_ASSERT(m_callbacks, get(node) != nullptr);
#if __GNUC__ >= 6
C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wnull-dereference")
#endif
id_type ich = get(node)->m_first_child;
#if __GNUC__ >= 6
C4_SUPPRESS_WARNING_GCC_POP
#endif
while(ich != NONE)
{
remove_children(ich);
@@ -1178,7 +1184,6 @@ id_type Tree::child_pos(id_type node, id_type ch) const
#if defined(__clang__)
# pragma clang diagnostic push
# pragma GCC diagnostic ignored "-Wnull-dereference"
#elif defined(__GNUC__)
# pragma GCC diagnostic push
# if __GNUC__ >= 6
@@ -1397,10 +1402,30 @@ id_type Tree::add_tag_directive(TagDirective const& td)
return pos;
}
namespace {
bool _create_tag_directive_from_str(csubstr directive_, TagDirective *td, Tree *tree)
{
_RYML_CB_CHECK(tree->callbacks(), directive_.begins_with("%TAG "));
if(!td->create_from_str(directive_))
{
_RYML_CB_ERR(tree->callbacks(), "invalid tag directive");
}
td->next_node_id = tree->size();
if(!tree->empty())
{
const id_type prev = tree->size() - 1;
if(tree->is_root(prev) && tree->type(prev) != NOTYPE && !tree->is_stream(prev))
++td->next_node_id;
}
_c4dbgpf("%TAG: handle={} prefix={} next_node={}", td->handle, td->prefix, td->next_node_id);
return true;
}
} // namespace
bool Tree::add_tag_directive(csubstr directive_)
{
TagDirective td;
if(td.create_from_str(directive_, this))
if(_create_tag_directive_from_str(directive_, &td, this))
{
add_tag_directive(td);
return true;
@@ -1833,5 +1858,121 @@ Tree::_lookup_path_token Tree::_next_token(lookup_result *r, _lookup_path_token
} // namespace c4
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
#include "c4/yml/event_handler_tree.hpp"
#include "c4/yml/parse_engine.def.hpp"
#include "c4/yml/parse.hpp"
namespace c4 {
namespace yml {
Location Tree::location(Parser const& parser, id_type node) const
{
// try hard to avoid getting the location from a null string.
Location loc;
if(_location_from_node(parser, node, &loc, 0))
return loc;
return parser.val_location(parser.source().str);
}
bool Tree::_location_from_node(Parser const& parser, id_type node, Location *C4_RESTRICT loc, id_type level) const
{
if(has_key(node))
{
csubstr k = key(node);
if(C4_LIKELY(k.str != nullptr))
{
_RYML_CB_ASSERT(m_callbacks, k.is_sub(parser.source()));
_RYML_CB_ASSERT(m_callbacks, parser.source().is_super(k));
*loc = parser.val_location(k.str);
return true;
}
}
if(has_val(node))
{
csubstr v = val(node);
if(C4_LIKELY(v.str != nullptr))
{
_RYML_CB_ASSERT(m_callbacks, v.is_sub(parser.source()));
_RYML_CB_ASSERT(m_callbacks, parser.source().is_super(v));
*loc = parser.val_location(v.str);
return true;
}
}
if(is_container(node))
{
if(_location_from_cont(parser, node, loc))
return true;
}
if(type(node) != NOTYPE && level == 0)
{
// try the prev sibling
{
const id_type prev = prev_sibling(node);
if(prev != NONE)
{
if(_location_from_node(parser, prev, loc, level+1))
return true;
}
}
// try the next sibling
{
const id_type next = next_sibling(node);
if(next != NONE)
{
if(_location_from_node(parser, next, loc, level+1))
return true;
}
}
// try the parent
{
const id_type parent = this->parent(node);
if(parent != NONE)
{
if(_location_from_node(parser, parent, loc, level+1))
return true;
}
}
}
return false;
}
bool Tree::_location_from_cont(Parser const& parser, id_type node, Location *C4_RESTRICT loc) const
{
_RYML_CB_ASSERT(m_callbacks, is_container(node));
if(!is_stream(node))
{
const char *node_start = _p(node)->m_val.scalar.str; // this was stored in the container
if(has_children(node))
{
id_type child = first_child(node);
if(has_key(child))
{
// when a map starts, the container was set after the key
csubstr k = key(child);
if(k.str && node_start > k.str)
node_start = k.str;
}
}
*loc = parser.val_location(node_start);
return true;
}
else // it's a stream
{
*loc = parser.val_location(parser.source().str); // just return the front of the buffer
}
return true;
}
} // namespace yml
} // namespace c4
C4_SUPPRESS_WARNING_GCC_CLANG_POP
C4_SUPPRESS_WARNING_MSVC_POP

15
src/c4/yml/tree.hpp
View File

@@ -777,6 +777,21 @@ public:
/** @} */
public:
/** @name locations */
/** @{ */
/** Get the location of a node from the parse used to parse this tree. */
Location location(Parser const& p, id_type node) const;
private:
bool _location_from_node(Parser const& p, id_type node, Location *C4_RESTRICT loc, id_type level) const;
bool _location_from_cont(Parser const& p, id_type node, Location *C4_RESTRICT loc) const;
/** @} */
public:
/** @name internal string arena */

66
src_extra/c4/yml/extra/event_handler_ints.cpp Normal file
View File

@@ -0,0 +1,66 @@
#ifdef RYML_SINGLE_HEADER_INTS
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_ints.hpp>
#endif
#elif defined(RYML_SINGLE_HEADER)
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_all.hpp>
#endif
#else
#include <c4/yml/parse_engine.def.hpp>
#endif
#ifndef _C4_YML_EXTRA_EVENT_HANDLER_INTS_HPP_
#include "c4/yml/extra/event_handler_ints.hpp"
#endif
namespace c4 {
namespace yml {
// instantiate the template
template class ParseEngine<extra::EventHandlerInts>;
namespace extra {
int32_t estimate_events_ints_size(csubstr src)
{
int32_t count = 7; // BSTR + BDOC + =VAL + EDOC + ESTR
for(size_t i = 0; i < src.len; ++i)
{
switch(src.str[i])
{
// this has strings preceding/following it
case ':':
case ',': // overestimate, assume map
count += 6;
break;
// these have (or are likely to have) a string following it
case '-':
case '&':
case '*':
case '<':
case '!':
case '\'':
case '"':
case '|':
case '>':
case '?':
case '\n':
count += 3;
break;
case '[':
case ']':
count += 4;
break;
case '{':
case '}':
count += 7;
break;
}
}
return count;
}
} // namespace extra
} // namespace yml
} // namespace c4

1315
src_extra/c4/yml/extra/event_handler_ints.hpp Normal file
View File

File diff suppressed because it is too large Load Diff

48
src_extra/c4/yml/extra/event_handler_testsuite.cpp Normal file
View File

@@ -0,0 +1,48 @@
#ifdef RYML_SINGLE_HEADER_INTS
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_ints.hpp>
#endif
#elif defined(RYML_SINGLE_HEADER)
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_all.hpp>
#endif
#else
#include <c4/yml/node.hpp>
#include <c4/yml/parse_engine.def.hpp>
#endif
#ifndef _C4_YML_EXTRA_EVENT_HANDLER_TESTSUITE_HPP_
#include "c4/yml/extra/event_handler_testsuite.hpp"
#endif
#ifndef _C4_YML_EXTRA_SCALAR_HPP_
#include "c4/yml/extra/scalar.hpp"
#endif
#ifndef _C4_YML_EXTRA_SCALAR_HPP_
#include "c4/yml/extra/string.hpp"
#endif
namespace c4 {
namespace yml {
namespace extra {
void append_scalar_escaped(extra::string *es, csubstr val)
{
size_t orig = es->size();
es->resize(es->capacity());
size_t sz = escape_scalar(substr(*es).sub(orig), val);
if (orig + sz > es->size())
{
es->resize(orig + sz);
sz = escape_scalar(substr(*es).sub(orig), val);
}
es->resize(orig + sz);
}
} // namespace extra
// instantiate the template
template class ParseEngine<extra::EventHandlerTestSuite>;
} // namespace yml
} // namespace c4

55
test/test_suite/test_suite_event_handler.hpp → src_extra/c4/yml/extra/event_handler_testsuite.hpp
View File

@@ -1,42 +1,38 @@
#ifndef _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_
#define _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_
#ifndef _C4_YML_EXTRA_EVENT_HANDLER_TESTSUITE_HPP_
#define _C4_YML_EXTRA_EVENT_HANDLER_TESTSUITE_HPP_
#ifdef RYML_SINGLE_HEADER
#include <ryml_all.hpp>
#else
#ifndef RYML_SINGLE_HEADER
#ifndef _C4_YML_EVENT_HANDLER_STACK_HPP_
#include "c4/yml/event_handler_stack.hpp"
#endif
#ifndef _C4_YML_DETAIL_PRINT_HPP_
#include "c4/yml/detail/print.hpp"
#endif
#endif
#ifndef _C4_YML_EXTRA_STRING_HPP_
#include "./string.hpp"
#include "c4/yml/extra/string.hpp"
#endif
C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast")
C4_SUPPRESS_WARNING_GCC("-Wuseless-cast")
namespace c4 {
namespace yml {
namespace extra {
/** @addtogroup doc_event_handlers
* @{ */
void append_escaped(extra::string *s, csubstr val);
/** The stack state needed specifically by @ref EventHandlerYamlStd */
struct EventHandlerYamlStdState : public ParserState
/** @cond dev */
struct EventHandlerTestSuiteState : public ParserState
{
NodeData ev_data;
};
void append_scalar_escaped(extra::string *s, csubstr val);
/** @endcond */
/** The event handler producing standard YAML events as used in the
/** This event produces standard YAML events as used in the
* [YAML test suite](https://github.com/yaml/yaml-test-suite).
* See the documentation for @ref doc_event_handlers, which has
* important notes about the event model used by rapidyaml.
@@ -46,14 +42,14 @@ struct EventHandlerYamlStdState : public ParserState
* playground](https://play.yaml.io/main/parser). It is not part of
* the library and is not installed. *
*/
struct EventHandlerYamlStd : public EventHandlerStack<EventHandlerYamlStd, EventHandlerYamlStdState>
struct EventHandlerTestSuite : public EventHandlerStack<EventHandlerTestSuite, EventHandlerTestSuiteState>
{
/** @name types
* @{ */
// our internal state must inherit from parser state
using state = EventHandlerYamlStdState;
using state = EventHandlerTestSuiteState;
using EventSink = extra::string;
@@ -82,13 +78,13 @@ public:
/** @name construction and resetting
* @{ */
EventHandlerYamlStd() : EventHandlerStack(), m_sink(), m_val_buffers(), m_key_tag_buf(), m_val_tag_buf(), m_tag_directives(), m_has_yaml_directive(), m_arena(), m_has_docs() {}
EventHandlerYamlStd(Callbacks const& cb) : EventHandlerStack(cb), m_sink(), m_val_buffers(), m_key_tag_buf(), m_val_tag_buf(), m_tag_directives(), m_has_yaml_directive(), m_arena(), m_has_docs() {}
EventHandlerYamlStd(EventSink *sink, Callbacks const& cb) : EventHandlerStack(cb), m_sink(sink), m_val_buffers(), m_key_tag_buf(), m_val_tag_buf(), m_tag_directives(), m_has_yaml_directive(), m_arena(), m_has_docs()
EventHandlerTestSuite() : EventHandlerStack(), m_sink(), m_val_buffers(), m_key_tag_buf(), m_val_tag_buf(), m_tag_directives(), m_has_yaml_directive(), m_arena(), m_has_docs() {}
EventHandlerTestSuite(Callbacks const& cb) : EventHandlerStack(cb), m_sink(), m_val_buffers(), m_key_tag_buf(), m_val_tag_buf(), m_tag_directives(), m_has_yaml_directive(), m_arena(), m_has_docs() {}
EventHandlerTestSuite(EventSink *sink, Callbacks const& cb) : EventHandlerStack(cb), m_sink(sink), m_val_buffers(), m_key_tag_buf(), m_val_tag_buf(), m_tag_directives(), m_has_yaml_directive(), m_arena(), m_has_docs()
{
reset();
}
EventHandlerYamlStd(EventSink *sink) : EventHandlerYamlStd(sink, get_callbacks()) {}
EventHandlerTestSuite(EventSink *sink) : EventHandlerTestSuite(sink, get_callbacks()) {}
void reset()
{
@@ -478,11 +474,11 @@ public:
C4_ALWAYS_INLINE void mark_key_scalar_unfiltered()
{
C4_NOT_IMPLEMENTED();
// nothing to do here
}
C4_ALWAYS_INLINE void mark_val_scalar_unfiltered()
{
C4_NOT_IMPLEMENTED();
// nothing to do here
}
/** @} */
@@ -495,8 +491,7 @@ public:
void set_key_anchor(csubstr anchor)
{
_c4dbgpf("node[{}]: set key anchor: [{}]~~~{}~~~", m_curr->node_id, anchor.len, anchor);
if(C4_UNLIKELY(_has_any_(KEYANCH)))
_RYML_CB_ERR_(m_stack.m_callbacks, "key cannot have both anchor and ref", m_curr->pos);
_RYML_CB_ASSERT(m_stack.m_callbacks, !_has_any_(KEYREF));
_RYML_CB_ASSERT(m_stack.m_callbacks, !anchor.begins_with('&'));
_enable_(KEYANCH);
m_curr->ev_data.m_key.anchor = anchor;
@@ -504,8 +499,7 @@ public:
void set_val_anchor(csubstr anchor)
{
_c4dbgpf("node[{}]: set val anchor: [{}]~~~{}~~~", m_curr->node_id, anchor.len, anchor);
if(C4_UNLIKELY(_has_any_(VALREF)))
_RYML_CB_ERR_(m_stack.m_callbacks, "val cannot have both anchor and ref", m_curr->pos);
_RYML_CB_ASSERT(m_stack.m_callbacks, !_has_any_(VALREF));
_RYML_CB_ASSERT(m_stack.m_callbacks, !anchor.begins_with('&'));
_enable_(VALANCH);
m_curr->ev_data.m_val.anchor = anchor;
@@ -710,7 +704,7 @@ public:
_send_key_props_();
_send_(' ');
_send_(scalar_type_code);
append_escaped(&_buf_(), scalar);
append_scalar_escaped(&_buf_(), scalar);
_send_('\n');
}
void _send_val_scalar_(csubstr scalar, char scalar_type_code)
@@ -719,7 +713,7 @@ public:
_send_val_props_();
_send_(' ');
_send_(scalar_type_code);
append_escaped(&_buf_(), scalar);
append_scalar_escaped(&_buf_(), scalar);
_send_('\n');
}
@@ -842,9 +836,10 @@ public:
/** @} */
} // namespace extra
} // namespace yml
} // namespace c4
C4_SUPPRESS_WARNING_GCC_POP
#endif /* _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ */
#endif /* _C4_YML_EVT_EXTRA_EVENT_HANDLER_TESTSUITE_HPP_ */

207
src_extra/c4/yml/extra/ints_to_testsuite.cpp Normal file
View File

@@ -0,0 +1,207 @@
#ifdef RYML_SINGLE_HEADER_INTS
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_ints.hpp>
#endif
#elif defined(RYML_SINGLE_HEADER)
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_all.hpp>
#endif
#endif
#ifndef _C4_YML_EXTRA_SCALAR_HPP_
#include "c4/yml/extra/scalar.hpp"
#endif
#ifndef _C4_YML_EXTRA_INTS_UTILS_HPP_
#include "c4/yml/extra/ints_utils.hpp"
#endif
#ifndef _C4_BITMASK_HPP_
#include "c4/bitmask.hpp"
#endif
C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wold-style-cast")
C4_SUPPRESS_WARNING_CLANG_WITH_PUSH("-Wold-style-cast")
// NOLINTBEGIN(hicpp-signed-bitwise)
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
namespace c4 {
namespace yml {
namespace extra {
size_t events_ints_to_testsuite(csubstr parsed_yaml,
csubstr arena,
ievt::DataType const* evts_ints,
ievt::DataType evts_ints_sz,
substr evts_test_suite)
{
auto getstr = [&](ievt::DataType i){
bool in_arena = evts_ints[i] & ievt::AREN;
csubstr region = !in_arena ? parsed_yaml : arena;
return region.sub((size_t)evts_ints[i+1], (size_t)evts_ints[i+2]);
};
size_t sz = 0;
auto append = [&](csubstr s){
size_t next = sz + s.len;
if (s.len && (next <= evts_test_suite.len && evts_test_suite.len))
memcpy(evts_test_suite.str + sz, s.str, s.len);
sz = next;
};
bool has_tag = false;
csubstr tag;
auto maybe_append_tag = [&]{
if(has_tag)
{
#ifdef RYML_NO_COVERAGE__TO_BE_DELETED
if(tag.begins_with('<'))
{
append(" ");
append(tag);
}
else
#endif
if(tag.begins_with("!<"))
{
append(" ");
append(tag.sub(1));
}
else if(tag.begins_with('!'))
{
append(" <");
append(tag);
append(">");
}
else
{
append(" <!");
append(tag);
append(">");
}
}
has_tag = false;
};
bool has_anchor = false;
csubstr anchor;
auto maybe_append_anchor = [&]{
if(has_anchor)
{
append(" &");
append(anchor);
}
has_anchor = false;
};
auto append_cont = [&](csubstr evt, csubstr style){
append(evt);
if(style.len)
{
append(" ");
append(style);
}
maybe_append_anchor();
maybe_append_tag();
append("\n");
};
auto append_val = [&](csubstr evt, csubstr val){
append("=VAL");
maybe_append_anchor();
maybe_append_tag();
append(" ");
append(evt);
substr buf = sz <= evts_test_suite.len ? evts_test_suite.sub(sz) : evts_test_suite.last(0);
sz += escape_scalar(buf, val);
append("\n");
};
for(ievt::DataType i = 0; i < evts_ints_sz; )
{
ievt::DataType evt = evts_ints[i];
if(evt & ievt::SCLR)
{
csubstr s = getstr(i);
if(evt & ievt::SQUO)
append_val("'", s);
else if(evt & ievt::DQUO)
append_val("\"", s);
else if(evt & ievt::LITL)
append_val("|", s);
else if(evt & ievt::FOLD)
append_val(">", s);
else //if(evt & ievt::PLAI)
append_val(":", s);
}
else if(evt & ievt::BSEQ)
{
if(evt & ievt::FLOW)
append_cont("+SEQ", "[]");
else
append_cont("+SEQ", "");
}
else if(evt & ievt::ESEQ)
{
append("-SEQ\n");
}
else if(evt & ievt::BMAP)
{
if(evt & ievt::FLOW)
append_cont("+MAP", "{}");
else
append_cont("+MAP", "");
}
else if(evt & ievt::EMAP)
{
append("-MAP\n");
}
else if(evt & ievt::ALIA)
{
append("=ALI *");
append(getstr(i));
append("\n");
}
else if(evt & ievt::TAG_)
{
has_tag = true;
tag = getstr(i);
}
else if(evt & ievt::ANCH)
{
has_anchor = true;
anchor = getstr(i);
}
else if(evt & ievt::BDOC)
{
if(evt & ievt::EXPL)
append("+DOC ---\n");
else
append("+DOC\n");
}
else if(evt & ievt::EDOC)
{
if(evt & ievt::EXPL)
append("-DOC ...\n");
else
append("-DOC\n");
}
else if(evt & ievt::BSTR)
{
append("+STR\n");
}
else if(evt & ievt::ESTR)
{
append("-STR\n");
}
i += (evt & ievt::WSTR) ? 3 : 1;
}
return sz;
}
} // namespace extra
} // namespace yml
} // namespace c4
// NOLINTEND(hicpp-signed-bitwise)
C4_SUPPRESS_WARNING_CLANG_POP
C4_SUPPRESS_WARNING_GCC_POP

69
src_extra/c4/yml/extra/ints_to_testsuite.hpp Normal file
View File

@@ -0,0 +1,69 @@
#ifndef _C4_YML_EXTRA_INTS_TO_TESTSUITE_HPP_
#define _C4_YML_EXTRA_INTS_TO_TESTSUITE_HPP_
#ifndef _C4_YML_EXTRA_EVENT_HANDLER_INTS_HPP_
#include "c4/yml/extra/event_handler_ints.hpp"
#endif
//-----------------------------------------------------------------------------
namespace c4 {
namespace yml {
namespace extra {
/** @addtogroup doc_event_handlers
* @{ */
/** Create a testsuite event string from integer events.
*
* This overload receives a buffer where the string should be written,
* and returns the size needed for the buffer. If that size is larger
* than the buffer's size, the user must resize the buffer and call
* again. */
RYML_EXPORT C4_NODISCARD
size_t events_ints_to_testsuite(csubstr parsed_yaml,
csubstr arena,
ievt::DataType const* evts_ints,
ievt::DataType evts_ints_sz,
substr evts_testsuite);
/** Create a testsuite event string from integer events, writing into
* an output container. */
template<class Container>
void events_ints_to_testsuite(csubstr parsed_yaml,
csubstr arena,
ievt::DataType const* evts_ints,
ievt::DataType evts_ints_sz,
Container *evts_testsuite)
{
size_t len = events_ints_to_testsuite(parsed_yaml, arena, evts_ints, evts_ints_sz, to_substr(*evts_testsuite));
if(len > evts_testsuite->size())
{
evts_testsuite->resize(len);
len = events_ints_to_testsuite(parsed_yaml, arena, evts_ints, evts_ints_sz, to_substr(*evts_testsuite));
}
evts_testsuite->resize(len);
}
/** Create a testsuite event string from integer events, returning a
* new container with the result. */
template<class Container>
Container events_ints_to_testsuite(csubstr parsed_yaml,
csubstr arena,
ievt::DataType const* evts_ints,
ievt::DataType evts_ints_sz)
{
Container ret;
events_ints_to_testsuite(parsed_yaml, arena, evts_ints, evts_ints_sz, &ret);
return ret;
}
/** @} */
} // namespace extra
} // namespace yml
} // namespace c4
#endif /* _C4_YML_EXTRA_INTS_TO_TESTSUITE_UTILS_HPP_ */

132
src_extra/c4/yml/extra/ints_utils.cpp Normal file
View File

@@ -0,0 +1,132 @@
#ifdef RYML_SINGLE_HEADER_INTS
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_ints.hpp>
#endif
#elif defined(RYML_SINGLE_HEADER)
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_all.hpp>
#endif
#endif
#ifndef _C4_YML_EXTRA_SCALAR_HPP_
#include "c4/yml/extra/scalar.hpp"
#endif
#ifndef _C4_YML_EXTRA_INTS_UTILS_HPP_
#include "c4/yml/extra/ints_utils.hpp"
#endif
#ifndef _C4_BITMASK_HPP_
#include "c4/bitmask.hpp"
#endif
C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wold-style-cast")
C4_SUPPRESS_WARNING_CLANG_WITH_PUSH("-Wold-style-cast")
// NOLINTBEGIN(hicpp-signed-bitwise)
namespace c4 {
template<>
c4::EnumSymbols<yml::extra::ievt::EventFlags> esyms<yml::extra::ievt::EventFlags>()
{
static constexpr const EnumSymbols<yml::extra::ievt::EventFlags>::Sym syms[] = {
{yml::extra::ievt::KEY_, "KEY_"},
{yml::extra::ievt::VAL_, "VAL_"},
{yml::extra::ievt::SCLR, "SCLR"},
{yml::extra::ievt::BSEQ, "BSEQ"},
{yml::extra::ievt::ESEQ, "ESEQ"},
{yml::extra::ievt::BMAP, "BMAP"},
{yml::extra::ievt::EMAP, "EMAP"},
{yml::extra::ievt::ALIA, "ALIA"},
{yml::extra::ievt::ANCH, "ANCH"},
{yml::extra::ievt::TAG_, "TAG_"},
{yml::extra::ievt::PLAI, "PLAI"},
{yml::extra::ievt::SQUO, "SQUO"},
{yml::extra::ievt::DQUO, "DQUO"},
{yml::extra::ievt::LITL, "LITL"},
{yml::extra::ievt::FOLD, "FOLD"},
{yml::extra::ievt::FLOW, "FLOW"},
{yml::extra::ievt::BLCK, "BLCK"},
{yml::extra::ievt::BDOC, "BDOC"},
{yml::extra::ievt::EDOC, "EDOC"},
{yml::extra::ievt::BSTR, "BSTR"},
{yml::extra::ievt::ESTR, "ESTR"},
{yml::extra::ievt::EXPL, "EXPL"},
{yml::extra::ievt::AREN, "AREN"},
{yml::extra::ievt::PSTR, "PSTR"},
{yml::extra::ievt::UNFILT, "UNFILT"},
{yml::extra::ievt::YAML, "YAML"},
{yml::extra::ievt::TAGD, "TAGD"},
{yml::extra::ievt::TAGV, "TAGV"},
};
return EnumSymbols<yml::extra::ievt::EventFlags>(syms);
}
namespace yml {
namespace extra {
namespace ievt {
size_t to_chars(substr buf, ievt::DataType flags)
{
return c4::bm2str<ievt::EventFlags>((flags & ievt::MASK), buf.str, buf.len);
}
csubstr to_chars_sub(substr buf, ievt::DataType flags)
{
size_t reqsize = ievt::to_chars(buf, flags);
RYML_CHECK(reqsize > 0u);
RYML_CHECK(reqsize < buf.len);
return buf.first(reqsize - 1u);
}
} // namespace ievt
} // namespace extra
} // namespace yml
} // namespace c4
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
namespace c4 {
namespace yml {
namespace extra {
void events_ints_print(csubstr parsed_yaml, csubstr arena, ievt::DataType const* evts, ievt::DataType evts_sz)
{
char buf[200];
for(ievt::DataType evtpos = 0, evtnumber = 0;
evtpos < evts_sz;
++evtnumber,
evtpos += ((evts[evtpos] & ievt::WSTR) ? 3 : 1))
{
ievt::DataType evt = evts[evtpos];
{
csubstr str = ievt::to_chars_sub(buf, evt);
printf("[%d][%d] %.*s(0x%x)", evtnumber, evtpos, (int)str.len, str.str, evt);
}
if (evt & ievt::WSTR)
{
bool in_arena = evt & ievt::AREN;
csubstr region = !in_arena ? parsed_yaml : arena;
bool safe = (evts[evtpos + 1] >= 0)
&& (evts[evtpos + 2] >= 0)
&& (evts[evtpos + 1] <= (int)region.len)
&& ((evts[evtpos + 1] + evts[evtpos + 2]) <= (int)region.len);
const char *str = safe ? (region.str + evts[evtpos + 1]) : "ERR!!!";
int len = safe ? evts[evtpos + 2] : 6;
printf(": %d [%d]~~~%.*s~~~", evts[evtpos+1], evts[evtpos+2], len, str);
if(in_arena)
printf(" (arenasz=%zu)", arena.len);
else
printf(" (srcsz=%zu)", parsed_yaml.len);
}
printf("\n");
}
}
} // namespace extra
} // namespace yml
} // namespace c4
// NOLINTEND(hicpp-signed-bitwise)
C4_SUPPRESS_WARNING_CLANG_POP
C4_SUPPRESS_WARNING_GCC_POP

38
src_extra/c4/yml/extra/ints_utils.hpp Normal file
View File

@@ -0,0 +1,38 @@
#ifndef _C4_YML_EXTRA_INTS_UTILS_HPP_
#define _C4_YML_EXTRA_INTS_UTILS_HPP_
#ifndef _C4_YML_EXTRA_EVENT_HANDLER_INTS_HPP_
#include "c4/yml/extra/event_handler_ints.hpp"
#endif
//-----------------------------------------------------------------------------
namespace c4 {
namespace yml {
namespace extra {
/** @addtogroup doc_event_handlers
* @{ */
namespace ievt {
/** Convert bit mask of @ref ievt::EventFlags to text. */
RYML_EXPORT size_t to_chars(substr buf, yml::extra::ievt::DataType flags);
/** Convert bit mask of @ref ievt::EventFlags to text. */
RYML_EXPORT csubstr to_chars_sub(substr buf, yml::extra::ievt::DataType flags);
} // namespace ievt
/** Print integer events to stdout */
RYML_EXPORT void events_ints_print(csubstr parsed_yaml, csubstr arena, ievt::DataType const* evts_ints, ievt::DataType evts_ints_sz);
/** @} */
} // namespace extra
} // namespace yml
} // namespace c4
#endif /* _C4_YML_EXTRA_INTS_UTILS_HPP_ */

39
test/test_suite/test_suite_event_handler.cpp → src_extra/c4/yml/extra/scalar.cpp
View File

@@ -1,23 +1,35 @@
#ifndef RYML_SINGLE_HEADER
#include <c4/yml/node.hpp>
#include <c4/yml/std/string.hpp>
#include <c4/yml/parse_engine.def.hpp>
#ifdef RYML_SINGLE_HEADER_INTS
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_ints.hpp>
#endif
#elif defined(RYML_SINGLE_HEADER)
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_all.hpp>
#endif
#endif
#ifndef _C4_YML_EXTRA_SCALAR_HPP_
#include <c4/yml/extra/scalar.hpp>
#endif
#include "./test_suite_event_handler.hpp"
namespace c4 {
namespace yml {
namespace extra {
// instantiate the template
template class ParseEngine<EventHandlerYamlStd>;
void append_escaped(extra::string *es, csubstr val)
size_t escape_scalar(substr buffer, csubstr val)
{
size_t pos = 0;
#define _append(repl) \
do { \
if(repl.len && (pos + repl.len <= buffer.len)) \
memcpy(buffer.str + pos, repl.str, repl.len); \
pos += repl.len; \
} while(0)
#define _c4flush_use_instead(i, repl, skip) \
do { \
es->append(val.range(prev, i)); \
es->append(repl); \
_append(val.range(prev, i)); \
_append(csubstr(repl)); \
prev = i + skip; \
} \
while(0)
@@ -72,9 +84,12 @@ void append_escaped(extra::string *es, csubstr val)
}
}
// flush the rest
es->append(val.sub(prev));
_append(val.sub(prev));
#undef _c4flush_use_instead
#undef _append
return pos;
}
} // namespace extra
} // namespace yml
} // namespace c4

23
src_extra/c4/yml/extra/scalar.hpp Normal file
View File

@@ -0,0 +1,23 @@
#ifndef _C4_YML_EXTRA_SCALAR_HPP_
#define _C4_YML_EXTRA_SCALAR_HPP_
#ifndef _C4_SUBSTR_HPP_
#include "c4/substr.hpp"
#endif
namespace c4 {
namespace yml {
namespace extra {
/** @addtogroup doc_event_handlers
* @{ */
size_t escape_scalar(substr s, csubstr val);
/** @} */
} // namespace extra
} // namespace yml
} // namespace c4
#endif /* _C4_YML_EVT_EXTRA_SCALAR_HPP_ */

7
test/test_suite/string.hpp → src_extra/c4/yml/extra/string.hpp
View File

@@ -1,11 +1,7 @@
#ifndef _C4_YML_EXTRA_STRING_HPP_
#define _C4_YML_EXTRA_STRING_HPP_
#ifdef RYML_SINGLE_HEADER
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_all.hpp>
#endif
#else
#ifndef RYML_SINGLE_HEADER
#ifndef _C4_YML_COMMON_HPP_
#include "c4/yml/common.hpp"
#endif
@@ -85,6 +81,7 @@ public:
public:
const char* data() const noexcept { return m_str; }
id_type size() const noexcept { return m_size; }
id_type capacity() const noexcept { return m_capacity; }

157
test/CMakeLists.txt
View File

@@ -1,15 +1,22 @@
c4_setup_testing(GTEST)
c4_add_executable(ryml-test-quickstart
SOURCES ../samples/quickstart.cpp
LIBS ryml
FOLDER test)
if(RYML_DBG)
target_compile_definitions(ryml-test-quickstart PRIVATE RYML_DBG)
endif()
c4_target_compile_flags(ryml-test-quickstart PUBLIC GCC -Wno-useless-cast)
c4_add_test(ryml-test-quickstart)
function(ryml_add_test_no_lib test_name)
c4_add_executable(${test_name} LIBS ryml FOLDER test ${ARGN})
if(RYML_DBG)
target_compile_definitions(${test_name} PRIVATE RYML_DBG)
c4_target_compile_flags(${test_name} PUBLIC GCC -Wno-useless-cast)
endif()
c4_add_test(${test_name})
endfunction()
ryml_add_test_no_lib(ryml-test-quickstart
SOURCES ../samples/quickstart.cpp)
ryml_add_test_no_lib(ryml-test-quickstart-ints
SOURCES
../samples/quickstart-ints.cpp
../src_extra/c4/yml/extra/event_handler_ints.hpp
../src_extra/c4/yml/extra/event_handler_ints.cpp
INC_DIRS ../src_extra)
c4_add_library(ryml-_testlib LIBRARY_TYPE STATIC
@@ -21,9 +28,28 @@ c4_add_library(ryml-_testlib LIBRARY_TYPE STATIC
test_lib/test_case.cpp
test_lib/test_engine.hpp
test_lib/test_engine.cpp
INC_DIRS ${CMAKE_CURRENT_LIST_DIR}
test_lib/test_events_ints_helpers.hpp
test_lib/test_events_ints_helpers.cpp
../src_extra/c4/yml/extra/event_handler_ints.cpp
../src_extra/c4/yml/extra/event_handler_ints.hpp
../src_extra/c4/yml/extra/event_handler_testsuite.cpp
../src_extra/c4/yml/extra/event_handler_testsuite.hpp
../src_extra/c4/yml/extra/ints_utils.cpp
../src_extra/c4/yml/extra/ints_utils.hpp
../src_extra/c4/yml/extra/ints_to_testsuite.cpp
../src_extra/c4/yml/extra/ints_to_testsuite.hpp
../src_extra/c4/yml/extra/scalar.hpp
../src_extra/c4/yml/extra/scalar.cpp
../src_extra/c4/yml/extra/string.hpp
INC_DIRS ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_LIST_DIR}/../src_extra
LIBS ryml c4fs gtest
FOLDER test)
if(BUILD_SHARED_LIBS)
# this is needed to resolve get_case() because it is provided by
# the test linking to this library
set_target_properties(ryml-_testlib PROPERTIES
POSITION_INDEPENDENT_CODE ON)
endif()
if(RYML_DBG)
target_compile_definitions(ryml-_testlib PUBLIC RYML_DBG)
endif()
@@ -55,30 +81,16 @@ function(ryml_add_test_case_group name)
endfunction()
c4_add_library(ryml-_evt_handler_yaml_std LIBRARY_STATIC
SOURCES
test_suite/test_suite_event_handler.cpp
test_suite/test_suite_event_handler.hpp
LIBS ryml
FOLDER test
)
if(RYML_DBG)
target_compile_definitions(ryml-_evt_handler_yaml_std PUBLIC RYML_DBG)
endif()
function(ryml_add_engine_test name)
ryml_add_test(${name} ryml-_evt_handler_yaml_std)
endfunction()
ryml_add_engine_test(parse_engine_1_doc)
ryml_add_engine_test(parse_engine_2_map)
ryml_add_engine_test(parse_engine_3_seq)
ryml_add_engine_test(parse_engine_4_anchor)
ryml_add_engine_test(parse_engine_5_tag)
ryml_add_engine_test(parse_engine_6_qmrk)
ryml_add_engine_test(parse_engine_7_seqimap)
ryml_add_engine_test(parse_engine_8_scalars_tokens)
ryml_add_engine_test(yaml_events)
ryml_add_test(engine_1_doc)
ryml_add_test(engine_2_map)
ryml_add_test(engine_3_seq)
ryml_add_test(engine_4_anchor)
ryml_add_test(engine_5_tag)
ryml_add_test(engine_6_qmrk)
ryml_add_test(engine_7_seqimap)
ryml_add_test(engine_8_scalars_tokens)
ryml_add_test(extra_testsuite)
ryml_add_test(extra_ints)
ryml_add_test(version)
ryml_add_test(callbacks)
ryml_add_test(stack)
@@ -175,20 +187,37 @@ if(RYML_TEST_TOOLS)
endfunction()
function(ryml_add_event_tool_test name expect_success flags contents)
ryml_create_file(${name}.yml "${contents}" file)
string(REPLACE " " ";" flags "${flags}")
add_test(NAME ryml-test-tool-events-${name} COMMAND ${RYML_TGT_EVENTS} ${flags} ${file})
if(NOT expect_success)
set_tests_properties(ryml-test-tool-events-${name} PROPERTIES WILL_FAIL TRUE)
endif()
endfunction()
ryml_get_target_exe(ryml-yaml-events RYML_TGT_EVENTS)
ryml_add_event_tool_test(success_tree TRUE "-t" "{foo: bar, baz: [exactly]}")
ryml_add_event_tool_test(success_evts TRUE "" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(fail_squo_tree FALSE "-t" "foo: 'bar")
ryml_add_event_tool_test(fail_squo_evts FALSE "" "foo: 'bar")
ryml_add_event_tool_test(fail_dquo_tree FALSE "-t" "foo: \"bar")
ryml_add_event_tool_test(fail_dquo_evts FALSE "" "foo: \"bar")
ryml_add_event_tool_test(fail_seq1_tree FALSE "-t" "[ a, b, c ] ]")
ryml_add_event_tool_test(fail_seq2_evts FALSE "" "[ [a, b, c ]")
ryml_add_event_tool_test(success_help TRUE "--help" "{foo: bar, baz: [exactly]}")
ryml_add_event_tool_test(success_help2 TRUE "tst --help" "{foo: bar, baz: [exactly]}")
ryml_add_event_tool_test(fail_nocmd FALSE "" "foo")
ryml_add_event_tool_test(fail_badcmd FALSE "barbar" "foo")
ryml_add_event_tool_test(success_timing TRUE "tst --timing" "{foo: bar, baz: [exactly]}")
ryml_add_event_tool_test(success_tree TRUE "tst" "{foo: bar, baz: [exactly]}")
ryml_add_event_tool_test(success_evts TRUE "tss" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(success_evts_ints TRUE "tsi" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(success_ryml_ints TRUE "ri" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(success_evts_ints_resize TRUE "tsi --ints-size 1" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(fail_evts_ints_resize FALSE "tsi --ints-size" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(fail_evts_ints_resize2 FALSE "tsi --ints-size xyz" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(fail_evts_ints_no_resize FALSE "tsi --ints-size 1 --ints-size-force" "{{this: is, a: keymap}: [seq,val]}")
ryml_add_event_tool_test(fail_squo_tree FALSE "tst" "foo: 'bar")
ryml_add_event_tool_test(fail_squo_evts FALSE "tss" "foo: 'bar")
ryml_add_event_tool_test(fail_dquo_tree FALSE "tst" "foo: \"bar")
ryml_add_event_tool_test(fail_dquo_evts FALSE "tss" "foo: \"bar")
ryml_add_event_tool_test(fail_seq1_tree FALSE "tst" "[ a, b, c ] ]")
ryml_add_event_tool_test(fail_seq2_evts FALSE "tss" "[ [a, b, c ]")
if(UNIX)
add_test(NAME ryml-test-tool-events-stdin COMMAND echo foo | ${RYML_TGT_EVENTS} tsi -)
endif()
add_test(NAME ryml-test-tool-events-nofile COMMAND ${RYML_TGT_EVENTS} tsi no_file)
set_tests_properties(ryml-test-tool-events-nofile PROPERTIES WILL_FAIL TRUE)
endif()
@@ -214,16 +243,26 @@ if(RYML_TEST_SUITE)
c4_add_executable(ryml-test-suite
SOURCES
test_suite.cpp
test_suite/test_suite_common.hpp
test_suite/test_suite_events_emitter.cpp
test_suite/test_suite_event_handler.cpp
test_suite/test_suite_event_handler.hpp
test_suite/test_suite_events.cpp
test_suite/test_suite_events.hpp
test_suite/test_suite_parts.cpp
test_suite/test_suite_parts.hpp
../src_extra/c4/yml/extra/event_handler_ints.cpp
../src_extra/c4/yml/extra/event_handler_ints.hpp
../src_extra/c4/yml/extra/event_handler_testsuite.cpp
../src_extra/c4/yml/extra/event_handler_testsuite.hpp
../src_extra/c4/yml/extra/ints_utils.cpp
../src_extra/c4/yml/extra/ints_utils.hpp
../src_extra/c4/yml/extra/ints_to_testsuite.cpp
../src_extra/c4/yml/extra/ints_to_testsuite.hpp
../src_extra/c4/yml/extra/scalar.cpp
../src_extra/c4/yml/extra/scalar.hpp
../src_extra/c4/yml/extra/string.hpp
testsuite.cpp
testsuite/testsuite_common.hpp
testsuite/testsuite_events_emitter.cpp
testsuite/testsuite_events.cpp
testsuite/testsuite_events.hpp
testsuite/testsuite_parts.cpp
testsuite/testsuite_parts.hpp
LIBS ryml-_testlib c4log
INC_DIRS ${CMAKE_CURRENT_LIST_DIR}/../src_extra
FOLDER test)
add_dependencies(ryml-test-build ryml-test-suite)
@@ -311,7 +350,7 @@ if(RYML_TEST_FUZZ)
test_fuzz/test_fuzz_${name}.cpp
test_fuzz/test_fuzz_main.cpp
${ARGN}
INC_DIRS ${CMAKE_CURRENT_LIST_DIR}
INC_DIRS ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_LIST_DIR}/../src_extra
LIBS ryml c4fs
FOLDER test/fuzz)
if(RYML_DBG)
@@ -330,7 +369,15 @@ if(RYML_TEST_FUZZ)
endforeach()
endfunction()
ryml_add_fuzz_test(parse_emit)
ryml_add_fuzz_test(events
../test/test_suite/test_suite_event_handler.hpp
../test/test_suite/test_suite_event_handler.cpp)
ryml_add_fuzz_test(events_testsuite
../src_extra/c4/yml/extra/event_handler_testsuite.hpp
../src_extra/c4/yml/extra/event_handler_testsuite.cpp
../src_extra/c4/yml/extra/scalar.hpp
../src_extra/c4/yml/extra/scalar.cpp
../src_extra/c4/yml/extra/string.hpp)
ryml_add_fuzz_test(events_ints
../src_extra/c4/yml/extra/event_handler_ints.hpp
../src_extra/c4/yml/extra/event_handler_ints.cpp
../src_extra/c4/yml/extra/scalar.hpp
../src_extra/c4/yml/extra/scalar.cpp)
endif()

0
test/test_parse_engine_1_doc.cpp → test/test_engine_1_doc.cpp
View File

0
test/test_parse_engine_2_map.cpp → test/test_engine_2_map.cpp
View File

0
test/test_parse_engine_3_seq.cpp → test/test_engine_3_seq.cpp
View File

31
test/test_parse_engine_4_anchor.cpp → test/test_engine_4_anchor.cpp
View File

@@ -1111,29 +1111,48 @@ ENGINE_TEST(DoubleAnchorKeyMap,
___(ps.end_stream());
}
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockMapKey_SU74, Location(2,10), ""
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockMapKey_SU74_1, Location(2,10), ""
"key1: &alias value1\n"
"&b *alias : value2\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockMapKey_SU74_2, Location(2,8), ""
"key1: &alias value1\n"
"*alias &b: value2\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowMapKey_SU74, Location(2,10), ""
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowMapKey_SU74_1, Location(2,10), ""
"{key1: &alias value1,\n"
"&b *alias : value2}\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowMapKey_SU74_2, Location(2,8), ""
"{key1: &alias value1,\n"
"*alias &b: value2}\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowMapVal_SU74, Location(2,18), ""
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowMapVal_SU74_1, Location(2,18), ""
"key1: &alias value1\n"
"value2: &b *alias\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowMapVal_SU74_2, Location(2,16), ""
"key1: &alias value1\n"
"value2: *alias &b\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockMapVal_SU74, Location(2,18), ""
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockMapVal_SU74_1, Location(2,18), ""
"{key1: &alias value1,\n"
"value2: &b *alias\n}")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockMapVal_SU74_2, Location(2,16), ""
"{key1: &alias value1,\n"
"value2: *alias &b\n}")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowSeqVal_SU74, Location(2,12), ""
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowSeqVal_SU74_1, Location(2,12), ""
"- &alias value1\n"
"- &b *alias\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsFlowSeqVal_SU74_2, Location(2,10), ""
"- &alias value1\n"
"- *alias &b\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockSeqVal_SU74, Location(2,10), ""
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockSeqVal_SU74_1, Location(2,10), ""
"[&alias value1,\n"
"&b *alias]\n")
ENGINE_TEST_ERRLOC(AnchorAndAliasAsBlockSeqVal_SU74_2, Location(2,8), ""
"[&alias value1,\n"
"*alias &b]\n")
} // namespace yml
} // namespace c4

32
test/test_parse_engine_5_tag.cpp → test/test_engine_5_tag.cpp
View File

@@ -710,6 +710,13 @@ ENGINE_TEST(DirectiveAndTag,
___(ps.end_stream());
}
ENGINE_TEST_ERR(TagCustomNotFound,
"--- !foo \"bar\"\n"
"...\n"
"%TAG !m! tag:example.com,2000:app/\n"
"--- !n!foo \"bar\"\n")
//-----------------------------------------------------------------------------
ENGINE_TEST_ERRLOC(TagTestSuiteU99R_0, Location(2,1),
@@ -1248,6 +1255,31 @@ ENGINE_TEST_ERR(DirectiveMultipleYAML_3,
"---\n"
"bar")
ENGINE_TEST_ERR(DirectiveWrong,
"--- !foo \"ba\"\n"
"...\n"
"%TAG ,,,,,\n"
"--- !foo \"bar\"\n")
ENGINE_TEST_ERR(DirectiveTooMany,
"--- !foo \"bar\"\n"
"...\n"
"%TAG ! tag:example.com,2000:app/\n"
"--- !foo \"bar\"\n"
"...\n"
"%TAG ! tag:example.com,2001:app/\n"
"...\n"
"--- !foo \"bar\"\n"
"...\n"
"%TAG ! tag:example.com,2002:app/\n"
"--- !foo \"bar\"\n"
"...\n"
"%TAG ! tag:example.com,2003:app/\n"
"--- !foo \"bar\"\n"
"...\n"
"%TAG ! tag:example.com,2004:app/\n"
"--- !foo \"bar\"\n")
//-----------------------------------------------------------------------------

0
test/test_parse_engine_6_qmrk.cpp → test/test_engine_6_qmrk.cpp
View File

0
test/test_parse_engine_7_seqimap.cpp → test/test_engine_7_seqimap.cpp
View File

24
test/test_parse_engine_8_scalars_tokens.cpp → test/test_engine_8_scalars_tokens.cpp
View File

@@ -6,6 +6,30 @@
namespace c4 {
namespace yml {
ENGINE_TEST(PlainScalarUnfiltered,
("foo: bar\n"),
"+STR\n"
"+DOC\n"
"+MAP\n"
"=VAL :foo\n"
"=VAL :bar\n"
"-MAP\n"
"-DOC\n"
"-STR\n")
{
___(ps.begin_stream());
___(ps.begin_doc());
___(ps.begin_map_val_block());
___(ps.mark_key_scalar_unfiltered());
___(ps.set_key_scalar_plain("foo"));
___(ps.mark_val_scalar_unfiltered());
___(ps.set_val_scalar_plain("bar"));
___(ps.end_map());
___(ps.end_doc());
___(ps.end_stream());
}
//-----------------------------------------------------------------------------
ENGINE_TEST(PlainScalarWithColon0,

10
test/test_explicit_key.cpp
View File

@@ -265,7 +265,7 @@ N(MFS, L{
})
);
ADD_CASE_TO_GROUP("M2N8", EXPECT_PARSE_ERROR,
ADD_CASE_TO_GROUP("M2N8", HAS_CONTAINER_KEYS,
R"(
- ? : x
- ? :
@@ -274,7 +274,7 @@ R"(
LineCol(2, 5)
);
ADD_CASE_TO_GROUP("explicit key 2nd", EXPECT_PARSE_ERROR,
ADD_CASE_TO_GROUP("explicit key 2nd", HAS_CONTAINER_KEYS,
R"(
a simple key: a value
? an explicit key: another value
@@ -282,7 +282,7 @@ a simple key: a value
LineCol(3, 19)
);
ADD_CASE_TO_GROUP("explicit key 1st", EXPECT_PARSE_ERROR,
ADD_CASE_TO_GROUP("explicit key 1st", HAS_CONTAINER_KEYS,
R"(
? an explicit key: another value
a simple key: a value
@@ -290,7 +290,7 @@ a simple key: a value
LineCol(2, 19)
);
ADD_CASE_TO_GROUP("explicit key nested in a map, 1st", EXPECT_PARSE_ERROR,
ADD_CASE_TO_GROUP("explicit key nested in a map, 1st", HAS_CONTAINER_KEYS,
R"(
map:
? an explicit key: another value
@@ -300,7 +300,7 @@ map:
LineCol(3, 21)
);
ADD_CASE_TO_GROUP("explicit key nested in a seq, 1st", EXPECT_PARSE_ERROR,
ADD_CASE_TO_GROUP("explicit key nested in a seq, 1st", HAS_CONTAINER_KEYS,
R"(
- ? an explicit key: another value
a simple key: a value

653
test/test_extra_ints.cpp Normal file
View File

@@ -0,0 +1,653 @@
#include "test_lib/test_case.hpp"
#include "test_lib/test_events_ints_helpers.hpp"
#include <c4/yml/extra/event_handler_ints.hpp>
// NOLINTBEGIN(hicpp-signed-bitwise)
namespace c4 {
namespace yml {
namespace extra {
struct IntEventsCase
{
const char *file;
const int line;
ParserOptions opts;
csubstr yaml;
const std::vector<IntEventWithScalar> evt;
void testeq(csubstr parsed_source, csubstr arena, ievt::DataType const* actual, size_t actual_size) const
{
RYML_TRACE_FMT("defined in:\n{}:{}: (here)\n", file, line);
#ifdef RYML_DBG
events_ints_print(parsed_source, arena, actual, (extra::ievt::DataType)actual_size);
#endif
test_events_ints_invariants(parsed_source, arena, actual, (ievt::DataType)actual_size);
test_events_ints(evt.data(), evt.size(), actual, actual_size, yaml, parsed_source, arena);
}
};
// this is required to work around a valgrind problem in gtest's
// printing of the test byte contents. We use the opportunity to print
// a line showing the location where the test case was defined.
std::ostream& operator<<(std::ostream& os, const IntEventsCase& e)
{
return os << "line[" << e.line << "]:\n" << e.file << ":" << e.line << ": (here)";
}
//-----------------------------------------------------------------------------
#define DECLARE_CSUBSTR_FROM_CHAR_ARR(name, ...) \
const char name##_[] = { __VA_ARGS__ }; \
csubstr name = {name##_, C4_COUNTOF(name##_)}
DECLARE_CSUBSTR_FROM_CHAR_ARR(dqesc_L6,
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x58, 0xa8),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x58, 0xa8),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x58, 0xa8),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x58, 0xa8),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x58, 0xa8),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x58, 0xa8),
);
DECLARE_CSUBSTR_FROM_CHAR_ARR(dqesc_P6,
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x57, 0xa9),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x57, 0xa9),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x57, 0xa9),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x57, 0xa9),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x57, 0xa9),
_RYML_CHCONST(-0x1e, 0xe2), _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x57, 0xa9),
);
using namespace ievt;
const bool needs_filter = true;
const IntEventsCase test_cases[] = {
// make the declarations shorter
#define e(...) IntEventWithScalar{__VA_ARGS__}
#define tc_(opts, ys, ...) IntEventsCase{__FILE__, __LINE__, opts, ys, std::initializer_list<IntEventWithScalar>(__VA_ARGS__)}
#define tc(ys, ...) tc_(ParserOptions{}, ys, __VA_ARGS__)
// case -------------------------------------------------
tc("!yamlscript/v0/bare\n--- !code\n42\n",
{
e(BSTR),
e(BDOC),
e(VAL_|TAG_, 0, 19, "!yamlscript/v0/bare"),
e(VAL_|SCLR|PLAI|PSTR, 0, 0, ""),
e(EDOC|PSTR),
e(BDOC|EXPL),
e(VAL_|TAG_, 24, 5, "!code"),
e(VAL_|SCLR|PLAI|PSTR, 30, 2, "42"),
e(EDOC|PSTR),
e(ESTR),
}),
// case -------------------------------------------------
tc("a: 1",
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 0, 1, "a"),
e(VAL_|SCLR|PLAI|PSTR, 3, 1, "1"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("say: 2 + 2",
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 0, 3, "say"),
e(VAL_|SCLR|PLAI|PSTR, 5, 5, "2 + 2"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("𝄞: ✅",
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 0, 4, "𝄞"),
e(VAL_|SCLR|PLAI|PSTR, 6, 3, "✅"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("[a, b, c]",
{
e(BSTR),
e(BDOC),
e(VAL_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 1, 1, "a"),
e(VAL_|SCLR|PLAI|PSTR, 4, 1, "b"),
e(VAL_|SCLR|PLAI|PSTR, 7, 1, "c"),
e(ESEQ|PSTR),
e(EDOC),
e(ESTR),
}),
// case ------------------------------
tc("[a: b]",
{
e(BSTR),
e(BDOC),
e(VAL_|BSEQ|FLOW),
e(VAL_|BMAP|FLOW),
e(KEY_|SCLR|PLAI, 1, 1, "a"),
e(VAL_|SCLR|PLAI|PSTR, 4, 1, "b"),
e(EMAP|PSTR),
e(ESEQ),
e(EDOC),
e(ESTR),
}),
// case ------------------------------
tc(""
"--- !yamlscript/v0\n"
"foo: !\n"
"- {x: y}\n"
"- [x, y]\n"
"- foo\n"
"- 'foo'\n"
"- \"foo\"\n"
"- |\n"
" foo\n"
"- >\n"
" foo\n"
"- [1, 2, true, false, null]\n"
"- &anchor-1 !tag-1 foobar\n"
"---\n"
"another: doc\n"
"",
{
e(BSTR),
e(BDOC|EXPL),
e(VAL_|TAG_, 4, 14, "!yamlscript/v0"),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 19, 3, "foo"),
e(VAL_|TAG_|PSTR, 24, 1, "!"),
e(VAL_|BSEQ|BLCK|PSTR),
e(VAL_|BMAP|FLOW),
e(KEY_|SCLR|PLAI, 29, 1, "x"),
e(VAL_|SCLR|PLAI|PSTR, 32, 1, "y"),
e(EMAP|PSTR),
e(VAL_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 38, 1, "x"),
e(VAL_|SCLR|PLAI|PSTR, 41, 1, "y"),
e(ESEQ|PSTR),
e(VAL_|SCLR|PLAI, 46, 3, "foo"),
e(VAL_|SCLR|SQUO|PSTR, 53, 3, "foo"),
e(VAL_|SCLR|DQUO|PSTR, 61, 3, "foo"),
e(VAL_|SCLR|LITL|PSTR, 70, 4, "foo\n", needs_filter),
e(VAL_|SCLR|FOLD|PSTR, 80, 4, "foo\n", needs_filter),
e(VAL_|BSEQ|FLOW|PSTR),
e(VAL_|SCLR|PLAI, 89, 1, "1"),
e(VAL_|SCLR|PLAI|PSTR, 92, 1, "2"),
e(VAL_|SCLR|PLAI|PSTR, 95, 4, "true"),
e(VAL_|SCLR|PLAI|PSTR, 101, 5, "false"),
e(VAL_|SCLR|PLAI|PSTR, 108, 4, "null"),
e(ESEQ|PSTR),
e(VAL_|TAG_, 126, 6, "!tag-1"),
e(VAL_|ANCH|PSTR, 117, 8, "anchor-1"),
e(VAL_|SCLR|PLAI|PSTR, 133, 6, "foobar"),
e(ESEQ|PSTR),
e(EMAP),
e(EDOC),
e(BDOC|EXPL),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 144, 7, "another"),
e(VAL_|SCLR|PLAI|PSTR, 153, 3, "doc"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("plain: well\n"
" a\n"
" b\n"
" c\n"
"squo: 'single''quote'\n"
"dquo: \"x\\t\\ny\"\n"
"lit: |\n"
" X\n"
" Y\n"
" Z\n"
"fold: >\n"
" U\n"
" V\n"
" W\n"
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 0, 5, "plain"),
e(VAL_|SCLR|PLAI|PSTR, 7, 10, "well a b c"),
e(KEY_|SCLR|PLAI|PSTR, 24, 4, "squo"),
e(VAL_|SCLR|SQUO|PSTR, 31, 12, "single'quote", needs_filter),
e(KEY_|SCLR|PLAI|PSTR, 46, 4, "dquo"),
e(VAL_|SCLR|DQUO|PSTR, 53, 4, "x\t\ny", needs_filter),
e(KEY_|SCLR|PLAI|PSTR, 61, 3, "lit"),
e(VAL_|SCLR|LITL|PSTR, 68, 6, "X\nY\nZ\n", needs_filter),
e(KEY_|SCLR|PLAI|PSTR, 89, 4, "fold"),
e(VAL_|SCLR|FOLD|PSTR, 97, 6, "U V W\n", needs_filter),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("- !!seq []",
{
e(BSTR),
e(BDOC),
e(VAL_|BSEQ|BLCK),
e(VAL_|TAG_, 2, 5, "!!seq"),
e(VAL_|BSEQ|FLOW|PSTR),
e(ESEQ),
e(ESEQ),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc(""
"defn run(prompt session=nil):\n"
" when session:\n"
" write session _ :append true: |+\n"
" Q: $(orig-prompt:trim)\n"
" A ($api-model):\n"
" $(answer:trim)\n"
""
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 0, 28, "defn run(prompt session=nil)"),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 32, 12, "when session"),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 50, 28, "write session _ :append true"),
e(VAL_|SCLR|LITL|PSTR, 83, 54, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n", needs_filter),
e(EMAP|PSTR),
e(EMAP),
e(EMAP),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("#!/usr/bin/env ys-0\n"
"\n"
"defn run(prompt session=nil):\n"
" session-text =:\n"
" when session && session:fs-e:\n"
"\n"
" answer =:\n"
" cond:\n"
" api-model =~ /^dall-e/:\n"
" openai-image(prompt).data.0.url\n"
" api-model.in?(anthropic-models):\n"
" anthropic(prompt):anthropic-message:format\n"
" api-model.in?(groq-models):\n"
" groq(prompt).choices.0.message.content:format\n"
" api-model.in?(openai-models):\n"
" openai-chat(prompt).choices.0.message.content:format\n"
" else: die()\n"
"\n"
" say: answer\n"
"\n"
" when session:\n"
" write session _ :append true: |+\n"
" Q: $(orig-prompt:trim)\n"
" A ($api-model):\n"
" $(answer:trim)\n"
"\n"
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 21, 28, "defn run(prompt session=nil)"),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 53, 14, "session-text ="),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 73, 28, "when session && session:fs-e"),
e(VAL_|SCLR|PLAI|PSTR, 0, 0, ""), // note empty scalar pointing at the front
e(EMAP|PSTR),
e(KEY_|SCLR|PLAI, 106, 8, "answer ="),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 120, 4, "cond"),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 132, 22, "api-model =~ /^dall-e/"),
e(VAL_|SCLR|PLAI|PSTR, 164, 31, "openai-image(prompt).data.0.url"),
e(KEY_|SCLR|PLAI|PSTR, 202, 31, "api-model.in?(anthropic-models)"),
e(VAL_|SCLR|PLAI|PSTR, 243, 42, "anthropic(prompt):anthropic-message:format"),
e(KEY_|SCLR|PLAI|PSTR, 292, 26, "api-model.in?(groq-models)"),
e(VAL_|SCLR|PLAI|PSTR, 328, 45, "groq(prompt).choices.0.message.content:format"),
e(KEY_|SCLR|PLAI|PSTR, 380, 28, "api-model.in?(openai-models)"),
e(VAL_|SCLR|PLAI|PSTR, 418, 52, "openai-chat(prompt).choices.0.message.content:format"),
e(KEY_|SCLR|PLAI|PSTR, 477, 4, "else"),
e(VAL_|SCLR|PLAI|PSTR, 483, 5, "die()"),
e(EMAP|PSTR),
e(EMAP),
e(KEY_|SCLR|PLAI, 492, 3, "say"),
e(VAL_|SCLR|PLAI|PSTR, 497, 6, "answer"),
e(KEY_|SCLR|PLAI|PSTR, 507, 12, "when session"),
e(VAL_|BMAP|BLCK|PSTR),
e(KEY_|SCLR|PLAI, 525, 28, "write session _ :append true"),
e(VAL_|SCLR|LITL|PSTR, 558, 55, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n", needs_filter),
e(EMAP|PSTR),
e(EMAP),
e(EMAP),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc(""
"---\n"
"plain: a\n"
" b\n"
"\n"
" c\n"
,
{
e(BSTR),
e(BDOC|EXPL),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|PLAI, 4, 5, "plain"),
e(VAL_|SCLR|PLAI|PSTR, 11, 5, "a b\nc"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("{key: map}: [seq, val]"
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|BMAP|FLOW),
e(KEY_|SCLR|PLAI, 1, 3, "key"),
e(VAL_|SCLR|PLAI|PSTR, 6, 3, "map"),
e(EMAP|PSTR),
e(VAL_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 13, 3, "seq"),
e(VAL_|SCLR|PLAI|PSTR, 18, 3, "val"),
e(ESEQ|PSTR),
e(EMAP),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("[key, seq]: {map: val}"
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 1, 3, "key"),
e(VAL_|SCLR|PLAI|PSTR, 6, 3, "seq"),
e(ESEQ|PSTR),
e(VAL_|BMAP|FLOW),
e(KEY_|SCLR|PLAI, 13, 3, "map"),
e(VAL_|SCLR|PLAI|PSTR, 18, 3, "val"),
e(EMAP|PSTR),
e(EMAP),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("[b,c]: d"
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 1, 1, "b"),
e(VAL_|SCLR|PLAI|PSTR, 3, 1, "c"),
e(ESEQ|PSTR),
e(VAL_|SCLR|PLAI, 7, 1, "d"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("[[b,c]]: d"
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|BSEQ|FLOW),
e(VAL_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 2, 1, "b"),
e(VAL_|SCLR|PLAI|PSTR, 4, 1, "c"),
e(ESEQ|PSTR),
e(ESEQ),
e(VAL_|SCLR|PLAI, 9, 1, "d"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("[ a, [ [[b,c]]: d, e]]: 23"
,
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 2, 1, "a"),
e(VAL_|BSEQ|FLOW|PSTR),
e(VAL_|BMAP|FLOW),
e(KEY_|BSEQ|FLOW),
e(VAL_|BSEQ|FLOW),
e(VAL_|SCLR|PLAI, 9, 1, "b"),
e(VAL_|SCLR|PLAI|PSTR, 11, 1, "c"),
e(ESEQ|PSTR),
e(ESEQ),
e(VAL_|SCLR|PLAI, 16, 1, "d"),
e(EMAP|PSTR),
e(VAL_|SCLR|PLAI, 19, 1, "e"),
e(ESEQ|PSTR),
e(ESEQ),
e(VAL_|SCLR|PLAI, 24, 2, "23"),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
// this requires the arena:
// every filtered \L or \P expands 2->3 bytes
tc("[\"\\L\\L\\L\\L\\L\\L\", \"\\P\\P\\P\\P\\P\\P\"]",
{
e(BSTR),
e(BDOC),
e(VAL_|BSEQ|FLOW),
e(VAL_|SCLR|DQUO|AREN, 0, 18, dqesc_L6),
e(VAL_|SCLR|DQUO|AREN|PSTR, 18, 18, dqesc_P6),
e(ESEQ|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
// this requires the arena: directives cause the tag to expand
tc("%YAML 1.2\n"
"---\n"
"- !light fluorescent\n"
"- !m something\n"
"...\n"
"%TAG !m! !extraordinary-\n"
"---\n"
"- !m!light green\n"
"- !!int 2\n"
,
{
e(BSTR),
e(YAML, 6, 3, "1.2"),
e(BDOC|EXPL|PSTR),
e(VAL_|BSEQ|BLCK),
e(VAL_|TAG_, 16, 6, "!light"),
e(VAL_|SCLR|PLAI|PSTR, 23, 11, "fluorescent"),
e(VAL_|TAG_|PSTR, 37, 2, "!m"),
e(VAL_|SCLR|PLAI|PSTR, 40, 9, "something"),
e(ESEQ|PSTR),
e(EDOC|EXPL),
e(TAGD, 59, 3, "!m!"),
e(TAGV|PSTR, 63, 15, "!extraordinary-"),
e(BDOC|EXPL|PSTR),
e(VAL_|BSEQ|BLCK),
e(VAL_|TAG_|AREN, 0, 20, "!extraordinary-light"),
e(VAL_|SCLR|PLAI|PSTR, 94, 5, "green"),
e(VAL_|TAG_|PSTR, 102, 5, "!!int"),
e(VAL_|SCLR|PLAI|PSTR, 108, 1, "2"),
e(ESEQ|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc("\"\\\"foo\\\"\": \"\\\"bar\\\"\"",
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|DQUO, 1, 5, "\"foo\""),
e(VAL_|SCLR|DQUO|PSTR, 12, 5, "\"bar\""),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
// case -------------------------------------------------
tc_(ParserOptions{}.scalar_filtering(false),
"\"\\\"foo\\\"\": \"\\\"bar\\\"\"",
{
e(BSTR),
e(BDOC),
e(VAL_|BMAP|BLCK),
e(KEY_|SCLR|DQUO|UNFILT, 1, 7, "\\\"foo\\\""),
e(VAL_|SCLR|DQUO|UNFILT|PSTR, 12, 7, "\\\"bar\\\""),
e(EMAP|PSTR),
e(EDOC),
e(ESTR),
}),
};
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
struct IntEventsTestHelper
{
IntEventsCase const& ec;
size_t required_size_expected;
size_t required_size_actual;
extra::EventHandlerInts handler;
ParseEngine<extra::EventHandlerInts> parser;
std::string src_copy;
std::vector<DataType> actual;
std::string arena;
IntEventsTestHelper(IntEventsCase const& ec_)
: ec(ec_)
, required_size_expected(num_ints(ec.evt.data(), ec.evt.size()))
, handler()
, parser(&handler, ec_.opts)
, src_copy()
, actual()
{
}
void run_with_size(size_t sz, size_t arena_sz)
{
src_copy.assign(ec.yaml.str, ec.yaml.len);
actual.resize(sz);
arena.resize(arena_sz);
handler.reset(to_substr(src_copy), to_substr(arena), actual.data(), (int)actual.size());
parser.parse_in_place_ev("(testyaml)", to_substr(src_copy));
required_size_actual = (size_t)handler.required_size_events();
}
};
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
struct IntEventsTest : public testing::TestWithParam<IntEventsCase>
{
// force an ascii name (some characters in the parameter are UTF8)
static std::string name2str(const testing::TestParamInfo<ParamType>& info)
{
std::string s = c4::catrs<std::string>("line_", info.param.line);
for (char &c : s)
if (!std::isalnum(c))
c = '_';
return s;
}
};
TEST_P(IntEventsTest, size_large_enough)
{
IntEventsTestHelper h(GetParam());
h.run_with_size(2u * h.required_size_expected, 2u * h.ec.yaml.len);
ASSERT_LT(h.required_size_actual, h.actual.size());
ASSERT_LT(h.handler.required_size_arena(), h.arena.size());
ASSERT_TRUE(h.handler.fits_buffers());
h.ec.testeq(to_csubstr(h.src_copy), to_csubstr(h.arena), h.actual.data(), h.required_size_actual);
}
TEST_P(IntEventsTest, size_too_small)
{
IntEventsTestHelper h(GetParam());
size_t small = h.required_size_expected / 2u;
h.run_with_size(small, 0u);
ASSERT_EQ(h.actual.size(), small);
ASSERT_EQ(h.arena.size(), 0u);
ASSERT_GT(h.required_size_actual, h.actual.size());
ASSERT_FALSE(h.handler.fits_buffers());
_c4dbgpf("retry! reqbuf={} reqarena={}", h.required_size_actual, h.handler.required_size_arena());
h.run_with_size(h.required_size_actual, h.handler.required_size_arena());
ASSERT_EQ(h.actual.size(), h.handler.required_size_events());
ASSERT_EQ(h.arena.size(), h.handler.required_size_arena());
ASSERT_TRUE(h.handler.fits_buffers());
h.ec.testeq(to_csubstr(h.src_copy), to_csubstr(h.arena), h.actual.data(), h.required_size_actual);
}
TEST_P(IntEventsTest, size_null)
{
IntEventsTestHelper h(GetParam());
h.run_with_size(0, 0);
ASSERT_GT(h.required_size_actual, h.actual.size());
h.run_with_size(h.required_size_actual, h.handler.required_size_arena());
ASSERT_TRUE(h.handler.fits_buffers());
h.ec.testeq(to_csubstr(h.src_copy), to_csubstr(h.arena), h.actual.data(), h.required_size_actual);
}
INSTANTIATE_TEST_SUITE_P(IntEvents, IntEventsTest, testing::ValuesIn(test_cases), &IntEventsTest::name2str);
} // namespace extra
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
// The other test executables are written to contain the declarative-style
// YmlTestCases. This executable does not have any but the build setup
// assumes it does, and links with the test lib, which requires an existing
// get_case() function. So this is here to act as placeholder until (if?)
// proper test cases are added here. This was detected in #47 (thanks
// @cburgard).
RYML_EXPORT Case const* get_case(csubstr)
{
return nullptr;
}
} // namespace yml
} // namespace c4
// NOLINTEND(hicpp-signed-bitwise)

12
test/test_yaml_events.cpp → test/test_extra_testsuite.cpp
View File

@@ -5,9 +5,9 @@
#include <gtest/gtest.h>
#include "./test_lib/test_case.hpp"
#include "./test_suite/test_suite_events.hpp"
#include "./test_suite/test_suite_event_handler.hpp"
#include "./test_suite/test_suite_events_emitter.cpp" // HACK
#include "./testsuite/testsuite_events.hpp"
#include "c4/yml/extra/event_handler_testsuite.hpp"
#include "./testsuite/testsuite_events_emitter.cpp" // HACK
namespace c4 {
namespace yml {
@@ -53,9 +53,9 @@ TEST_P(EventsTest, from_parser)
EventsCase const& ec = GetParam();
printf("%s:%d: %s", ec.file, ec.line, ec.name.str);
RYML_TRACE_FMT("defined in:\n{}:{}: {}", ec.file, ec.line, ec.name);
EventHandlerYamlStd::EventSink sink;
EventHandlerYamlStd handler(&sink);
ParseEngine<EventHandlerYamlStd> parser(&handler);
extra::EventHandlerTestSuite::EventSink sink;
extra::EventHandlerTestSuite handler(&sink);
ParseEngine<extra::EventHandlerTestSuite> parser(&handler);
std::string src_copy(ec.src.str, ec.src.len);
parser.parse_in_place_ev("(testyaml)", to_substr(src_copy));
csubstr result = sink;

1
test/test_filter.cpp
View File

@@ -4,6 +4,7 @@
#include "c4/yml/filter_processor.hpp"
#include "c4/yml/event_handler_tree.hpp"
#include "c4/yml/parse_engine.hpp"
#include "c4/yml/parse_engine.def.hpp"
#include "c4/yml/parse.hpp"
#include "c4/yml/tree.hpp"
#include "c4/yml/node.hpp"

38
test/test_fuzz/test_fuzz_common.hpp
View File

@@ -11,8 +11,9 @@
#include <c4/yml/event_handler_tree.hpp>
#include <c4/yml/parse_engine.def.hpp>
#endif
#include <test_suite/test_suite_events.hpp>
#include <test_suite/test_suite_event_handler.hpp>
#include <testsuite/testsuite_events.hpp>
#include <c4/yml/extra/event_handler_ints.hpp>
#include <c4/yml/extra/event_handler_testsuite.hpp>
#include <cstdio>
#ifdef C4_EXCEPTIONS
@@ -109,9 +110,9 @@ inline int fuzztest_yaml_events(uint32_t case_number, csubstr src)
{
C4_UNUSED(case_number);
set_callbacks(create_custom_callbacks());
EventHandlerYamlStd::EventSink sink = {};
EventHandlerYamlStd handler(&sink, create_custom_callbacks());
ParseEngine<EventHandlerYamlStd> parser(&handler);
extra::EventHandlerTestSuite::EventSink sink = {};
extra::EventHandlerTestSuite handler(&sink, create_custom_callbacks());
ParseEngine<extra::EventHandlerTestSuite> parser(&handler);
std::string str(src.begin(), src.end());
C4_IF_EXCEPTIONS_(try, if(setjmp(jmp_env) == 0))
{
@@ -129,6 +130,33 @@ inline int fuzztest_yaml_events(uint32_t case_number, csubstr src)
return 0;
}
inline int fuzztest_yaml_events_ints(uint32_t case_number, csubstr src)
{
C4_UNUSED(case_number);
set_callbacks(create_custom_callbacks());
using Handler = extra::EventHandlerInts;
Handler handler{};
ParseEngine<extra::EventHandlerInts> parser(&handler);
std::string str(src.begin(), src.end());
std::vector<char> arena(str.size());
std::vector<Handler::value_type> event_ints;
event_ints.reserve(256);
handler.reset(to_substr(str), to_substr(arena), event_ints.data(), static_cast<Handler::value_type>(event_ints.size()));
C4_IF_EXCEPTIONS_(try, if(setjmp(jmp_env) == 0))
{
_if_dbg(_dbg_printf("in[{}]: [{}]~~~\n{}\n~~~\n", case_number, src.len, src); fflush(NULL));
parser.parse_in_place_ev("input", c4::to_substr(str));
C4_DONT_OPTIMIZE(event_ints);
}
C4_IF_EXCEPTIONS_(catch(std::exception const&), else)
{
// if an exception leaks from here, it is likely because of a greedy noexcept
_if_dbg(fprintf(stdout, "err\n"); fflush(NULL));
return 1;
}
return 0;
}
} // namespace yml
} // namespace c4

9
test/test_fuzz/test_fuzz_events_ints.cpp Normal file
View File

@@ -0,0 +1,9 @@
#include "./test_fuzz_common.hpp"
#include <atomic>
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *str, size_t len)
{
static std::atomic<uint32_t> case_number{0};
c4::csubstr src = {reinterpret_cast<const char*>(str), len};
return c4::yml::fuzztest_yaml_events_ints(case_number++, src);
}

0
test/test_fuzz/test_fuzz_events.cpp → test/test_fuzz/test_fuzz_events_testsuite.cpp
View File

6
test/test_github_issues.cpp
View File

@@ -736,9 +736,9 @@ R"(
LineCol(4, 15)
);
ADD_CASE_TO_GROUP("github128/1", RESOLVE_REFS | EXPECT_PARSE_ERROR, "a: *invalid");
ADD_CASE_TO_GROUP("github128/2", RESOLVE_REFS | EXPECT_PARSE_ERROR, "*");
ADD_CASE_TO_GROUP("github128/3", RESOLVE_REFS | EXPECT_PARSE_ERROR, "*abc");
ADD_CASE_TO_GROUP("github128/1", RESOLVE_REFS | EXPECT_RESOLVE_ERROR, "a: *invalid");
ADD_CASE_TO_GROUP("github128/2", RESOLVE_REFS | EXPECT_RESOLVE_ERROR, "*");
ADD_CASE_TO_GROUP("github128/3", RESOLVE_REFS | EXPECT_RESOLVE_ERROR, "*abc");
ADD_CASE_TO_GROUP("github128/4", "*abc", N(VAL, "*abc", AR(VALREF, "*abc")));
ADD_CASE_TO_GROUP("github129", RESOLVE_REFS, R"(

7
test/test_json.cpp
View File

@@ -11,6 +11,11 @@
#include <gtest/gtest.h>
C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
#if __GNUC__ >= 6
C4_SUPPRESS_WARNING_GCC("-Wnull-dereference")
#endif
namespace foo {
template<class T>
@@ -735,3 +740,5 @@ Case const* get_case(csubstr /*name*/)
} // namespace yml
} // namespace c4
C4_SUPPRESS_WARNING_GCC_CLANG_POP

2
test/test_lib/callbacks_tester.hpp
View File

@@ -4,7 +4,7 @@
#ifndef RYML_SINGLE_HEADER
#include "c4/yml/common.hpp"
#include <c4/charconv.hpp>
#include "c4/yml/detail/parser_dbg.hpp"
#include "c4/yml/detail/dbgprint.hpp"
#else
#endif
#include <vector>

12
test/test_lib/test_case.cpp
View File

@@ -715,18 +715,14 @@ CaseData* get_data(csubstr name)
{
std::string tmp;
replace_all("\r", "", c->src, &tmp);
cd->unix_style.src_buf.assign(tmp.begin(), tmp.end());
cd->unix_style.src = to_substr(cd->unix_style.src_buf);
cd->unix_style_json.src_buf.assign(tmp.begin(), tmp.end());
cd->unix_style_json.src = to_substr(cd->unix_style.src_buf);
cd->unix_style.assign(to_csubstr(tmp));
cd->unix_style_json.assign(to_csubstr(tmp));
}
{
std::string tmp;
replace_all("\n", "\r\n", cd->unix_style.src, &tmp);
cd->windows_style.src_buf.assign(tmp.begin(), tmp.end());
cd->windows_style.src = to_substr(cd->windows_style.src_buf);
cd->windows_style_json.src_buf.assign(tmp.begin(), tmp.end());
cd->windows_style_json.src = to_substr(cd->windows_style.src_buf);
cd->windows_style.assign(to_csubstr(tmp));
cd->windows_style_json.assign(to_csubstr(tmp));
}
}
else

22
test/test_lib/test_case.hpp
View File

@@ -8,7 +8,7 @@
#include "c4/std/string.hpp"
#include "c4/format.hpp"
#include <c4/yml/yml.hpp>
#include <c4/yml/detail/parser_dbg.hpp>
#include <c4/yml/detail/dbgprint.hpp>
#include <c4/yml/detail/print.hpp>
#endif
@@ -269,10 +269,11 @@ struct ExpectError
typedef enum {
EXPECT_PARSE_ERROR = (1<<0),
RESOLVE_REFS = (1<<1),
JSON_WRITE = (1<<2), // TODO: make it the opposite: opt-out instead of opt-in
JSON_READ = (1<<3),
HAS_CONTAINER_KEYS = (1<<4),
HAS_MULTILINE_SCALAR = (1<<5),
EXPECT_RESOLVE_ERROR = (1<<2),
JSON_WRITE = (1<<3), // TODO: make it the opposite: opt-out instead of opt-in
JSON_READ = (1<<4),
HAS_CONTAINER_KEYS = (1<<5),
HAS_MULTILINE_SCALAR = (1<<6),
} TestCaseFlags_e;
@@ -299,6 +300,13 @@ struct Case
// a persistent data store to avoid repeating operations on every test
struct CaseDataLineEndings
{
void assign(csubstr src_orig)
{
parse_buf_ints.assign(src_orig.begin(), src_orig.end());
src_buf.assign(src_orig.begin(), src_orig.end());
src = to_substr(src_buf);
}
std::vector<char> src_buf;
substr src;
@@ -323,6 +331,10 @@ struct CaseDataLineEndings
Tree emitted_tree_json;
Tree recreated;
std::string parse_buf_ints;
std::vector<int> parsed_ints;
std::vector<char> arena_ints;
};

3
test/test_lib/test_case_node.cpp
View File

@@ -4,6 +4,9 @@
C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
C4_SUPPRESS_WARNING_GCC("-Wuseless-cast")
#if __GNUC__ >= 6
C4_SUPPRESS_WARNING_GCC("-Wnull-dereference")
#endif
namespace c4 {
namespace yml {

312
test/test_lib/test_engine.cpp
View File

@@ -1,4 +1,7 @@
#include "./test_engine.hpp"
#include "testsuite/testsuite_events.hpp"
#include "c4/yml/extra/ints_utils.hpp"
#include "c4/yml/extra/ints_to_testsuite.hpp"
@@ -38,13 +41,24 @@ std::vector<std::string> inject_comments(std::string const& src_)
} // anon
void test_expected_error_events_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location)
void test_expected_error_testsuite_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location)
{
ExpectError::check_error([&]{
EventHandlerYamlStd::EventSink sink;
EventHandlerYamlStd handler(&sink);
extra::EventHandlerTestSuite::EventSink sink;
extra::EventHandlerTestSuite handler(&sink);
handler.reset();
ParseEngine<EventHandlerYamlStd> parser(&handler);
ParseEngine<extra::EventHandlerTestSuite> parser(&handler);
std::string copy = parsed_yaml;
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
}, expected_error_location);
}
void test_expected_error_ints_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location)
{
ExpectError::check_error([&]{
extra::EventHandlerInts handler{};
handler.reset(to_csubstr(parsed_yaml), substr{}, nullptr, 0);
ParseEngine<extra::EventHandlerInts> parser(&handler);
std::string copy = parsed_yaml;
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
}, expected_error_location);
@@ -61,15 +75,17 @@ void test_expected_error_tree_from_yaml(std::string const& parsed_yaml, Location
ASSERT_EQ(&tree, parser.m_evt_handler->m_tree);
std::string copy = parsed_yaml;
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
tree.resolve_tags();
}, expected_error_location);
}
void test_new_parser_events_from_yaml(ReferenceYaml const& yaml, std::string const& expected_events)
void test_engine_testsuite_from_yaml(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts)
{
EventHandlerYamlStd::EventSink sink;
EventHandlerYamlStd handler(&sink);
extra::EventHandlerTestSuite::EventSink sink;
extra::EventHandlerTestSuite handler(&sink);
handler.reset();
ParseEngine<EventHandlerYamlStd> parser(&handler);
ParseEngine<extra::EventHandlerTestSuite> parser(&handler, opts);
std::string copy = yaml.parsed;
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
csubstr result = sink;
@@ -77,32 +93,95 @@ void test_new_parser_events_from_yaml(ReferenceYaml const& yaml, std::string con
EXPECT_EQ(std::string(result.str, result.len), expected_events);
}
void test_new_parser_tree_from_yaml(ReferenceYaml const& yaml)
void test_engine_ints_from_yaml(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts)
{
extra::EventHandlerInts handler{};
using IntType = extra::ievt::DataType;
//NOTE! crashes in MIPS64 Debug c++20 (but not c++11) when size is 0:
//std::vector<IntType> actual_evts(empty.size());
std::vector<IntType> actual_evts; // DO THIS!
size_t size_reference = num_ints(yaml.expected_ints.data(), yaml.expected_ints.size());
int size_estimated = extra::estimate_events_ints_size(to_csubstr(yaml.parsed));
// there was an error in gcc<5 where the copy buffer was NOT
// assigned when using a std::string:
//std::string copy = yaml.parsed; gcc<5 ERROR, see below
std::vector<char> copy(yaml.parsed.begin(), yaml.parsed.end());
_c4dbgpf("parsing: [{}]{}", copy.size(), c4::fmt::hex(copy.data()));
std::vector<char> arena(copy.size());
handler.reset(to_csubstr(copy), to_substr(arena), actual_evts.data(), (IntType)actual_evts.size());
ParseEngine<extra::EventHandlerInts> parser(&handler, opts);
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
EXPECT_GE(size_estimated, handler.required_size_events());
if(yaml.expected_ints_enabled)
{
EXPECT_EQ(size_reference, handler.required_size_events());
}
size_t sz = (size_t)handler.required_size_events();
if (!handler.fits_buffers())
{
if(sz > actual_evts.size())
actual_evts.resize(sz);
copy.assign(yaml.parsed.begin(), yaml.parsed.end());
if(handler.required_size_arena() > arena.size())
arena.resize(handler.required_size_arena());
_c4dbgpf("parsing again: (before) [{}]{}", copy.size(), c4::fmt::hex(copy.data()));
//copy = yaml.parsed; ERROR: bad assignment in gcc<5
_c4dbgpf("parsing again: (after) [{}]{}", copy.size(), c4::fmt::hex(copy.data()));
handler.reset(to_csubstr(copy), to_substr(arena), actual_evts.data(), (IntType)actual_evts.size());
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
}
actual_evts.resize(sz);
#ifdef RYML_DBG
extra::events_ints_print(to_csubstr(copy), to_csubstr(arena), actual_evts.data(), (IntType)actual_evts.size());
#endif
{
RYML_TRACE_FMT("invariants", 0);
extra::test_events_ints_invariants(to_csubstr(copy), to_csubstr(arena), actual_evts.data(), (IntType)actual_evts.size());
}
if (yaml.expected_ints_enabled)
{
RYML_TRACE_FMT("here", 0);
test_events_ints(yaml.expected_ints.data(), yaml.expected_ints.size(),
actual_evts.data(), actual_evts.size(),
to_csubstr(yaml.parsed), to_csubstr(copy), to_csubstr(arena));
}
{
RYML_TRACE_FMT("cmp", 0);
std::string actual_test_suite_evts = extra::events_ints_to_testsuite<std::string>(to_csubstr(copy), to_csubstr(arena), actual_evts.data(), (IntType)actual_evts.size());
test_compare_events(to_csubstr(expected_events),
to_csubstr(actual_test_suite_evts),
/*ignore_doc_style*/false,
/*ignore_container_style*/false,
/*ignore_scalar_style*/false,
/*ignore_tag_normalization*/true);
}
}
void test_engine_tree_from_yaml(ReferenceYaml const& yaml, ParserOptions opts)
{
if(yaml.test_case_flags & HAS_CONTAINER_KEYS)
{
test_expected_error_tree_from_yaml(yaml.parsed, yaml.expected_error_location);
return;
}
else
{
Tree tree = {};
EventHandlerTree handler(&tree, tree.root_id());
ASSERT_EQ(&tree, handler.m_tree);
ParseEngine<EventHandlerTree> parser(&handler);
ASSERT_EQ(&handler, parser.m_evt_handler);
ASSERT_EQ(&tree, parser.m_evt_handler->m_tree);
std::string copy = yaml.parsed;
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
#ifdef RYML_DBG
print_tree(tree);
#endif
std::string actual = emitrs_yaml<std::string>(tree);
_c4dbgpf("~~~\n{}~~~\n", actual);
EXPECT_EQ(actual, yaml.emitted);
}
Tree tree = {};
EventHandlerTree handler(&tree, tree.root_id());
ASSERT_EQ(&tree, handler.m_tree);
ParseEngine<EventHandlerTree> parser(&handler, opts);
ASSERT_EQ(&handler, parser.m_evt_handler);
ASSERT_EQ(&tree, parser.m_evt_handler->m_tree);
std::string copy = yaml.parsed;
parser.parse_in_place_ev("(testyaml)", to_substr(copy));
#ifdef RYML_DBG
print_tree(tree);
#endif
std::string actual = emitrs_yaml<std::string>(tree);
_c4dbgpf("~~~\n{}~~~\n", actual);
EXPECT_EQ(actual, yaml.emitted);
}
void test_new_parser_events_from_yaml_with_comments(ReferenceYaml const& yaml, std::string const& expected_events)
void test_engine_testsuite_from_yaml_with_comments(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts)
{
if(yaml.test_case_flags & HAS_CONTAINER_KEYS)
return;
@@ -117,11 +196,28 @@ void test_new_parser_events_from_yaml_with_comments(ReferenceYaml const& yaml, s
SCOPED_TRACE(transformed_str);
SCOPED_TRACE("commented");
transformed.parsed = transformed_str;
test_new_parser_events_from_yaml(transformed, expected_events);
test_engine_testsuite_from_yaml(transformed, expected_events, opts);
}
}
void test_new_parser_tree_from_yaml_with_comments(ReferenceYaml const& yaml)
void test_engine_ints_from_yaml_with_comments(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts)
{
if(yaml.test_case_flags & HAS_MULTILINE_SCALAR)
return;
ReferenceYaml transformed = yaml;
const auto injected_comments = inject_comments(yaml.parsed);
for(size_t i = 0; i < injected_comments.size(); ++i)
{
const auto & transformed_str = injected_comments[i];
_c4dbgpf("transformed[{}/{}]=~~~[{}]\n{}\n~~~", i, injected_comments.size(), transformed_str.size(), to_csubstr(transformed_str));
SCOPED_TRACE(transformed_str);
SCOPED_TRACE("commented");
transformed.parsed = transformed_str;
test_engine_ints_from_yaml(transformed, expected_events, opts);
}
}
void test_engine_tree_from_yaml_with_comments(ReferenceYaml const& yaml, ParserOptions opts)
{
if(yaml.test_case_flags & HAS_CONTAINER_KEYS)
return;
@@ -136,7 +232,163 @@ void test_new_parser_tree_from_yaml_with_comments(ReferenceYaml const& yaml)
SCOPED_TRACE(transformed_str);
SCOPED_TRACE("commented");
transformed.parsed = transformed_str;
test_new_parser_tree_from_yaml(transformed);
test_engine_tree_from_yaml(transformed, opts);
}
}
csubstr parse_anchor_and_tag(csubstr tokens, OptionalScalar *anchor, OptionalScalar *tag)
{
*anchor = OptionalScalar{};
*tag = OptionalScalar{};
if(tokens.begins_with('&'))
{
size_t pos = tokens.first_of(' ');
if(pos == (size_t)csubstr::npos)
{
*anchor = tokens.sub(1);
tokens = {};
}
else
{
*anchor = tokens.first(pos).sub(1);
tokens = tokens.right_of(pos);
}
_c4dbgpf("anchor: {}", anchor->get());
}
if(tokens.begins_with('<'))
{
size_t pos = tokens.find('>');
RYML_ASSERT(pos != (size_t)csubstr::npos);
*tag = tokens.first(pos + 1);
tokens = tokens.right_of(pos).triml(' ');
_c4dbgpf("tag: {}", tag->maybe_get());
}
return tokens;
}
void test_compare_events(csubstr ref_evts,
csubstr emt_evts,
bool ignore_doc_style,
bool ignore_container_style,
bool ignore_scalar_style,
bool ignore_tag_normalization)
{
RYML_TRACE_FMT("actual=~~~\n{}~~~", emt_evts);
RYML_TRACE_FMT("expected=~~~\n{}~~~", ref_evts);
auto test_anchor_tag = [&](csubstr &ref, csubstr &emt){
OptionalScalar reftag = {}, refanchor = {};
OptionalScalar emttag = {}, emtanchor = {};
ref = parse_anchor_and_tag(ref, &refanchor, &reftag).triml(' ');
emt = parse_anchor_and_tag(emt, &emtanchor, &emttag).triml(' ');
EXPECT_EQ(bool(refanchor), bool(emtanchor));
if(bool(refanchor) && bool(emtanchor))
{
EXPECT_EQ(refanchor.get(), emtanchor.get());
}
EXPECT_EQ(bool(reftag), bool(emttag));
if(reftag && emttag)
{
if(!ignore_tag_normalization)
{
EXPECT_EQ(reftag.get(), emttag.get());
}
}
};
auto test_doc = [&](csubstr ref, csubstr emt){
EXPECT_TRUE(ref.begins_with("+DOC") || ref.begins_with("-DOC"));
EXPECT_TRUE(emt.begins_with("+DOC") || emt.begins_with("-DOC"));
EXPECT_EQ(emt.begins_with("+DOC"), ref.begins_with("+DOC"));
EXPECT_EQ(emt.begins_with("-DOC"), ref.begins_with("-DOC"));
if(ignore_doc_style)
{
if(ref.begins_with("+DOC"))
{
ref = ref.stripr("---").trimr(' ');
emt = emt.stripr("---").trimr(' ');
}
else
{
ASSERT_TRUE(ref.begins_with("-DOC"));
ref = ref.stripr("...").trimr(' ');
emt = emt.stripr("...").trimr(' ');
}
}
EXPECT_EQ(ref, emt);
};
auto test_container = [&](csubstr ref, csubstr emt){
RYML_TRACE_FMT("expected={}", ref);
RYML_TRACE_FMT("actual={}", emt);
EXPECT_TRUE(ref.begins_with("+MAP") || ref.begins_with("+SEQ"));
EXPECT_TRUE(emt.begins_with("+MAP") || emt.begins_with("+SEQ"));
EXPECT_EQ(emt.begins_with("+MAP"), ref.begins_with("+MAP"));
EXPECT_EQ(emt.begins_with("+SEQ"), ref.begins_with("+SEQ"));
csubstr rest_ref = ref.sub(4).triml(' ');
csubstr rest_emt = emt.sub(4).triml(' ');
if(rest_ref.begins_with("{}"))
{
if(!ignore_container_style)
{
EXPECT_TRUE(rest_emt.begins_with("{}"));
}
}
else if(rest_ref.begins_with("[]"))
{
if(!ignore_container_style)
{
EXPECT_TRUE(rest_emt.begins_with("[]"));
}
}
test_anchor_tag(ref, emt);
};
auto test_val_with_scalar_wildcard = [&](csubstr ref, csubstr emt){
ASSERT_TRUE(ref.begins_with("=VAL "));
ASSERT_TRUE(emt.begins_with("=VAL "));
ref = ref.sub(5);
emt = emt.sub(5);
test_anchor_tag(ref, emt);
ASSERT_GE(ref.len, 0);
ASSERT_GE(emt.len, 0);
EXPECT_TRUE(ref[0] == ':' || ref[0] == '\'' || ref[0] == '"' || ref[0] == '|' || ref[0] == '>');
EXPECT_TRUE(emt[0] == ':' || emt[0] == '\'' || emt[0] == '"' || emt[0] == '|' || emt[0] == '>');
if(!ignore_scalar_style)
{
EXPECT_EQ(ref[0], emt[0]);
}
ref = ref.sub(1);
emt = emt.sub(1);
EXPECT_EQ(ref, emt);
};
EXPECT_EQ(bool(ref_evts.len), bool(emt_evts.len));
size_t posref = 0;
size_t posemt = 0;
while(posref < ref_evts.len && posemt < emt_evts.len)
{
const size_t endref = ref_evts.find('\n', posref);
const size_t endemt = emt_evts.find('\n', posemt);
ASSERT_FALSE((endref == npos || endemt == npos) && (endref != endemt));
csubstr ref = ref_evts.range(posref, endref);
csubstr emt = emt_evts.range(posemt, endemt);
if(ref != emt)
{
if(ref.begins_with("+DOC") || emt.begins_with("-DOC"))
{
test_doc(ref, emt);
}
else if(ref.begins_with("+MAP") || ref.begins_with("+SEQ"))
{
test_container(ref, emt);
}
else if(ref.begins_with("=VAL"))
{
test_val_with_scalar_wildcard(ref, emt);
}
else
{
ASSERT_EQ(ref, emt);
}
}
posref = endref + 1u;
posemt = endemt + 1u;
}
}

347
test/test_lib/test_engine.hpp
View File

@@ -11,7 +11,9 @@
#endif
#include <gtest/gtest.h>
#include "./test_lib/test_case.hpp"
#include "./test_suite/test_suite_event_handler.hpp"
#include "./test_lib/test_events_ints_helpers.hpp"
#include "c4/yml/extra/event_handler_testsuite.hpp"
#include "c4/yml/extra/event_handler_ints.hpp"
namespace c4 {
@@ -20,27 +22,37 @@ namespace yml {
struct ReferenceYaml
{
ReferenceYaml( std::string const& s ) : test_case_flags(), expected_error_location(), parsed(s), emitted(s) {}
ReferenceYaml( std::string const& p, std::string const& e) : test_case_flags(), expected_error_location(), parsed(p), emitted(e) {}
ReferenceYaml(TestCaseFlags_e tf, std::string const& p, std::string const& e) : test_case_flags(tf), expected_error_location(), parsed(p), emitted(e) {}
ReferenceYaml(TestCaseFlags_e tf, std::string const& p ) : test_case_flags(tf), expected_error_location(), parsed(p), emitted(p) {}
ReferenceYaml( Location linecol_, std::string const& p ) : test_case_flags(), expected_error_location(linecol_), parsed(p), emitted(p) { RYML_ASSERT(linecol_); }
ReferenceYaml(TestCaseFlags_e tf, Location linecol_, std::string const& p, std::string const& e) : test_case_flags(tf), expected_error_location(linecol_), parsed(p), emitted(e) { RYML_ASSERT(linecol_); }
ReferenceYaml(TestCaseFlags_e tf, Location linecol_, std::string const& p ) : test_case_flags(tf), expected_error_location(linecol_), parsed(p), emitted(p) { RYML_ASSERT(linecol_); }
ReferenceYaml( std::string const& s ) : test_case_flags(), expected_error_location(), parsed(s), emitted(s), expected_ints(), expected_ints_enabled(false) {}
ReferenceYaml( std::string const& p, std::string const& e) : test_case_flags(), expected_error_location(), parsed(p), emitted(e), expected_ints(), expected_ints_enabled(false) {}
ReferenceYaml(TestCaseFlags_e tf, std::string const& p, std::string const& e) : test_case_flags(tf), expected_error_location(), parsed(p), emitted(e), expected_ints(), expected_ints_enabled(false) {}
ReferenceYaml(TestCaseFlags_e tf, std::string const& p ) : test_case_flags(tf), expected_error_location(), parsed(p), emitted(p), expected_ints(), expected_ints_enabled(false) {}
ReferenceYaml( Location linecol_, std::string const& p ) : test_case_flags(), expected_error_location(linecol_), parsed(p), emitted(p), expected_ints(), expected_ints_enabled(false) { RYML_ASSERT(linecol_); }
ReferenceYaml(TestCaseFlags_e tf, Location linecol_, std::string const& p, std::string const& e) : test_case_flags(tf), expected_error_location(linecol_), parsed(p), emitted(e), expected_ints(), expected_ints_enabled(false) { RYML_ASSERT(linecol_); }
ReferenceYaml(TestCaseFlags_e tf, Location linecol_, std::string const& p ) : test_case_flags(tf), expected_error_location(linecol_), parsed(p), emitted(p), expected_ints(), expected_ints_enabled(false) { RYML_ASSERT(linecol_); }
ReferenceYaml( std::string const& s , std::vector<extra::IntEventWithScalar> ints) : test_case_flags(), expected_error_location(), parsed(s), emitted(s), expected_ints(std::move(ints)), expected_ints_enabled(true) {}
ReferenceYaml( std::string const& p, std::string const& e, std::vector<extra::IntEventWithScalar> ints) : test_case_flags(), expected_error_location(), parsed(p), emitted(e), expected_ints(std::move(ints)), expected_ints_enabled(true) {}
ReferenceYaml(TestCaseFlags_e tf, std::string const& p, std::string const& e, std::vector<extra::IntEventWithScalar> ints) : test_case_flags(tf), expected_error_location(), parsed(p), emitted(e), expected_ints(std::move(ints)), expected_ints_enabled(true) {}
ReferenceYaml(TestCaseFlags_e tf, std::string const& p , std::vector<extra::IntEventWithScalar> ints) : test_case_flags(tf), expected_error_location(), parsed(p), emitted(p), expected_ints(std::move(ints)), expected_ints_enabled(true) {}
ReferenceYaml( Location linecol_, std::string const& p , std::vector<extra::IntEventWithScalar> ints) : test_case_flags(), expected_error_location(linecol_), parsed(p), emitted(p), expected_ints(std::move(ints)), expected_ints_enabled(true) { RYML_ASSERT(linecol_); }
ReferenceYaml(TestCaseFlags_e tf, Location linecol_, std::string const& p, std::string const& e, std::vector<extra::IntEventWithScalar> ints) : test_case_flags(tf), expected_error_location(linecol_), parsed(p), emitted(e), expected_ints(std::move(ints)), expected_ints_enabled(true) { RYML_ASSERT(linecol_); }
ReferenceYaml(TestCaseFlags_e tf, Location linecol_, std::string const& p , std::vector<extra::IntEventWithScalar> ints) : test_case_flags(tf), expected_error_location(linecol_), parsed(p), emitted(p), expected_ints(std::move(ints)), expected_ints_enabled(true) { RYML_ASSERT(linecol_); }
TestCaseFlags_e test_case_flags;
Location expected_error_location;
std::string parsed;
std::string emitted;
std::vector<extra::IntEventWithScalar> expected_ints;
bool expected_ints_enabled;
};
template<template<class> class EventProducerFn>
C4_NO_INLINE void test_new_parser_str_from_events(std::string const& expected_events)
C4_NO_INLINE void test_engine_str_from_testsuite(std::string const& expected_events)
{
EventHandlerYamlStd::EventSink sink;
EventHandlerYamlStd handler(&sink);
extra::EventHandlerTestSuite::EventSink sink;
extra::EventHandlerTestSuite handler(&sink);
handler.reset();
EventProducerFn<EventHandlerYamlStd> event_producer;
EventProducerFn<extra::EventHandlerTestSuite> event_producer;
event_producer(handler);
csubstr result = sink;
_c4dbgpf("~~~\n{}~~~\n", result);
@@ -48,7 +60,7 @@ C4_NO_INLINE void test_new_parser_str_from_events(std::string const& expected_ev
}
template<template<class> class EventProducerFn>
C4_NO_INLINE void test_new_parser_tree_from_events(ReferenceYaml const& yaml)
C4_NO_INLINE void test_engine_tree_from_testsuite(ReferenceYaml const& yaml)
{
if(yaml.test_case_flags & HAS_CONTAINER_KEYS)
{
@@ -74,17 +86,43 @@ C4_NO_INLINE void test_new_parser_tree_from_events(ReferenceYaml const& yaml)
}
}
void test_new_parser_events_from_yaml(ReferenceYaml const& yaml, std::string const& expected_events);
void test_new_parser_tree_from_yaml(ReferenceYaml const& yaml);
void test_new_parser_events_from_yaml_with_comments(ReferenceYaml const& yaml, std::string const& expected_events);
void test_new_parser_tree_from_yaml_with_comments(ReferenceYaml const& yaml);
void test_expected_error_events_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location={});
void test_expected_error_tree_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location={});
void test_engine_testsuite_from_yaml(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts);
void test_engine_ints_from_yaml(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts);
void test_engine_tree_from_yaml(ReferenceYaml const& yaml, ParserOptions opts);
void test_engine_testsuite_from_yaml_with_comments(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts);
void test_engine_ints_from_yaml_with_comments(ReferenceYaml const& yaml, std::string const& expected_events, ParserOptions opts);
void test_engine_tree_from_yaml_with_comments(ReferenceYaml const& yaml, ParserOptions opts);
void test_expected_error_testsuite_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location={});
void test_expected_error_ints_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location={});
void test_expected_error_tree_from_yaml(std::string const& parsed_yaml, Location const& expected_error_location={});
//-----------------------------------------------------------------------------
struct OptionalScalar
{
csubstr val = {};
bool was_set = false;
operator csubstr() const { return get(); }
operator bool() const { return was_set; }
void operator= (csubstr v) { val = v; was_set = true; }
csubstr get() const { RYML_ASSERT(was_set); return val; }
csubstr maybe_get() const { return was_set ? val : csubstr(""); }
};
csubstr parse_anchor_and_tag(csubstr tokens, OptionalScalar *anchor, OptionalScalar *tag);
void test_compare_events(csubstr ref_evts,
csubstr emt_evts,
bool ignore_doc_style,
bool ignore_container_style,
bool ignore_scalar_style,
bool ignore_tag_normalization);
//-----------------------------------------------------------------------------
#ifdef RYML_DBG
#define _RYML_SHOWFILELINE(name) printf("%s:%d: " #name "\n", __FILE__, __LINE__)
@@ -98,11 +136,20 @@ void test_expected_error_tree_from_yaml(std::string const& parsed_yaml, Location
#define ENGINE_TEST_ERRLOC(name, location, refyaml) \
\
\
TEST(EngineTest, name##_err_events_from_yaml) \
TEST(EngineTest, name##_err_testsuite_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_err_events_from_yaml"); \
test_expected_error_events_from_yaml(refyaml, location); \
SCOPED_TRACE(#name "_err_testsuite_from_yaml"); \
test_expected_error_testsuite_from_yaml(refyaml, location); \
_RYML_SHOWFILELINE(name); \
} \
\
\
TEST(EngineTest, name##_err_ints_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_err_ints_from_yaml"); \
test_expected_error_ints_from_yaml(refyaml, location); \
_RYML_SHOWFILELINE(name); \
} \
\
@@ -118,107 +165,135 @@ TEST(EngineTest, name##_err_tree_from_yaml) \
//-----------------------------------------------------------------------------
#define ENGINE_TEST_ERR(name, refyaml) \
\
\
TEST(EngineTest, name##_err_events_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_err_events_from_yaml"); \
test_expected_error_events_from_yaml(refyaml); \
_RYML_SHOWFILELINE(name); \
} \
\
\
TEST(EngineTest, name##_err_tree_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_err_tree_from_yaml"); \
test_expected_error_tree_from_yaml(refyaml); \
_RYML_SHOWFILELINE(name); \
#define ENGINE_TEST_ERR(name, refyaml) \
\
\
TEST(EngineTest, name##_err_testsuite_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_err_testsuite_from_yaml"); \
test_expected_error_testsuite_from_yaml(refyaml); \
_RYML_SHOWFILELINE(name); \
} \
\
\
TEST(EngineTest, name##_err_ints_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_err_ints_from_yaml"); \
test_expected_error_ints_from_yaml(refyaml); \
_RYML_SHOWFILELINE(name); \
} \
\
\
TEST(EngineTest, name##_err_tree_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_err_tree_from_yaml"); \
test_expected_error_tree_from_yaml(refyaml); \
_RYML_SHOWFILELINE(name); \
}
//-----------------------------------------------------------------------------
/* declare a common parser test for the existing event handlers */ \
/* declare a parse engine test for the existing event handlers */
#define ENGINE_TEST(name, refyaml, events) \
\
\
/* declare a function that will produce a \
sequence of events */ \
template<class Ps> \
void name##_impl(Ps &ps); \
\
\
/* package the function into a class */ \
template<class Ps> \
struct name \
{ \
void operator() (Ps &ps) \
{ \
name##_impl(ps); \
} \
}; \
\
\
TEST(EngineTest, name##_str_from_events) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_str_from_events"); \
test_new_parser_str_from_events<name>(events); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_tree_from_events) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_wtree_from_events"); \
ReferenceYaml yaml refyaml; \
test_new_parser_tree_from_events<name>(yaml); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_events_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_events_from_yaml"); \
ReferenceYaml yaml refyaml; \
test_new_parser_events_from_yaml(yaml, events); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_tree_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_wtree_from_yaml"); \
ReferenceYaml yaml refyaml; \
test_new_parser_tree_from_yaml(yaml); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_events_from_yaml_with_comments) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_events_from_yaml_with_comments"); \
ReferenceYaml yaml refyaml; \
test_new_parser_events_from_yaml_with_comments(yaml, events); \
} \
\
TEST(EngineTest, name##_tree_from_yaml_with_comments) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_tree_from_yaml"); \
ReferenceYaml yaml refyaml; \
test_new_parser_tree_from_yaml_with_comments(yaml); \
_RYML_SHOWFILELINE(name); \
} \
\
\
/* define the function that will produce the \
* sequence of events */ \
template<class Ps> \
ENGINE_TEST_(name, ParserOptions{}, refyaml, events)
#define ENGINE_TEST_(name, opts, refyaml, events) \
\
\
/* declare a function that will produce a \
sequence of events */ \
template<class Ps> \
void name##_impl(Ps &ps); \
\
\
/* package the function into a class */ \
template<class Ps> \
struct name \
{ \
void operator() (Ps &ps) \
{ \
name##_impl(ps); \
} \
}; \
\
\
TEST(EngineTest, name##_str_from_testsuite) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_str_from_testsuite"); \
test_engine_str_from_testsuite<name>(events); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_tree_from_testsuite) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_tree_from_testsuite"); \
ReferenceYaml yaml refyaml; \
test_engine_tree_from_testsuite<name>(yaml); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_testsuite_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_testsuite_from_yaml"); \
ReferenceYaml yaml refyaml; \
test_engine_testsuite_from_yaml(yaml, events, opts); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_ints_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_event_ints_from_yaml"); \
ReferenceYaml yaml refyaml; \
test_engine_ints_from_yaml(yaml, events, opts); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_tree_from_yaml) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_tree_from_yaml"); \
ReferenceYaml yaml refyaml; \
test_engine_tree_from_yaml(yaml, opts); \
_RYML_SHOWFILELINE(name); \
} \
\
TEST(EngineTest, name##_testsuite_from_yaml_with_comments) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_testsuite_from_yaml_with_comments"); \
ReferenceYaml yaml refyaml; \
test_engine_testsuite_from_yaml_with_comments(yaml, events, opts); \
} \
\
TEST(EngineTest, name##_ints_from_yaml_with_comments) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_ints_from_yaml_with_comments"); \
ReferenceYaml yaml refyaml; \
test_engine_ints_from_yaml_with_comments(yaml, events, opts); \
} \
\
TEST(EngineTest, name##_tree_from_yaml_with_comments) \
{ \
_RYML_SHOWFILELINE(name); \
SCOPED_TRACE(#name "_tree_from_yaml"); \
ReferenceYaml yaml refyaml; \
test_engine_tree_from_yaml_with_comments(yaml, opts); \
_RYML_SHOWFILELINE(name); \
} \
\
\
/* define the function that will produce the \
* sequence of events */ \
template<class Ps> \
void name##_impl(Ps &ps)
@@ -228,7 +303,25 @@ void name##_impl(Ps &ps)
#if !defined(RYML_DBG)
#define ___(stmt) stmt
#else
inline void _print_handler_info(EventHandlerYamlStd const& ps, csubstr stmt, const char *file, int line)
#define ___(stmt) \
do \
{ \
stmt; \
_print_handler_info(ps, #stmt, __FILE__, __LINE__); \
} while(0)
inline C4_NO_INLINE void _print_handler_info(EventHandlerTree const& ps, csubstr stmt, const char *file, int line)
{
if(ps.m_parent)
_dbg_printf("{}:{}: parent.id={} curr.id={} {}\n",
file, line, ps.m_parent->node_id, ps.m_curr->node_id, stmt);
else
_dbg_printf("{}:{}: parent.id=-- curr.id={} {}\n",
file, line, ps.m_curr->node_id, stmt);
print_tree(*ps.m_tree);
}
inline C4_NO_INLINE void _print_handler_info(extra::EventHandlerTestSuite const& ps, csubstr stmt, const char *file, int line)
{
_dbg_printf("{}:{}: {}", file, line, stmt);
auto indent = [](id_type n){
@@ -249,22 +342,12 @@ inline void _print_handler_info(EventHandlerYamlStd const& ps, csubstr stmt, con
}
}
}
inline void _print_handler_info(EventHandlerTree const& ps, csubstr stmt, const char *file, int line)
inline C4_NO_INLINE void _print_handler_info(extra::EventHandlerInts const& ps, csubstr stmt, const char *file, int line)
{
if(ps.m_parent)
_dbg_printf("{}:{}: parent.id={} curr.id={} {}\n",
file, line, ps.m_parent->node_id, ps.m_curr->node_id, stmt);
else
_dbg_printf("{}:{}: parent.id=-- curr.id={} {}\n",
file, line, ps.m_curr->node_id, stmt);
print_tree(*ps.m_tree);
_dbg_printf("{}:{}: {}", file, line, stmt);
(void)ps;
}
#define ___(stmt) \
do \
{ \
stmt; \
_print_handler_info(ps, #stmt, __FILE__, __LINE__); \
} while(0)
#endif
} // namespace yml

440
test/test_lib/test_events_ints_helpers.cpp Normal file
View File

@@ -0,0 +1,440 @@
#include "test_lib/test_case.hpp"
#include "./test_events_ints_helpers.hpp"
// NOLINTBEGIN(hicpp-signed-bitwise)
namespace c4 {
namespace yml {
namespace extra {
size_t num_ints(IntEventWithScalar const *evt, size_t evt_size)
{
size_t sz = 0;
for(size_t i = 0; i < evt_size; ++i)
sz += evt[i].required_size();
return sz;
}
void test_events_ints(IntEventWithScalar const* expected, size_t expected_sz,
ievt::DataType const* actual, size_t actual_sz,
csubstr yaml,
csubstr parsed_source,
csubstr arena)
{
int status = true;
size_t num_ints_expected = num_ints(expected, expected_sz);
EXPECT_EQ(actual_sz, num_ints_expected);
status = (actual_sz == num_ints_expected);
char actualbuf[100];(void)actualbuf;
char expectedbuf[100];(void)expectedbuf;
for(size_t ia = 0, ie = 0; ie < expected_sz; ++ie)
{
EXPECT_LT(ia, actual_sz);
if (ia >= actual_sz)
break;
#define _test_eq(lhs, rhs, fmt, ...) \
do \
{ \
_c4dbgpf("status={} cmp={} ie={} ia={}: {}={} == {}={} " fmt, \
status, (lhs == rhs), ie, ia, #lhs, lhs, rhs, #rhs, __VA_ARGS__); \
status &= int(lhs == rhs); \
EXPECT_EQ(lhs, rhs); \
} while(0)
csubstr sactual = ievt::to_chars_sub(actualbuf, actual[ia]);
csubstr sexpect = ievt::to_chars_sub(expectedbuf, expected[ie].flags);
_test_eq(actual[ia], expected[ie].flags, "", 0);
_test_eq(sactual, sexpect, "", 0);
if((expected[ie].flags & ievt::WSTR) && (actual[ia] & ievt::WSTR))
{
_test_eq(expected[ie].str_start, actual[ia + 1], "", 0);
_test_eq(expected[ie].str_len, actual[ia + 2], "", 0);
bool in_arena = actual[ia] & ievt::AREN;
bool safeexpected = !in_arena ?
(expected[ie].str_start < (int)parsed_source.len && expected[ie].str_start + expected[ie].str_len <= (int)parsed_source.len)
:
(expected[ie].str_start < (int)arena.len && expected[ie].str_start + expected[ie].str_len <= (int)arena.len);
bool safeactual = !in_arena ?
(ia + 2 < actual_sz) && (actual[ia + 1] < (int)parsed_source.len && actual[ia + 1] + actual[ia + 2] <= (int)parsed_source.len)
:
(ia + 2 < actual_sz) && (actual[ia + 1] < (int)arena.len && actual[ia + 1] + actual[ia + 2] <= (int)arena.len) ;
_test_eq(safeactual, true, "", 0);
_test_eq(safeactual, safeexpected, "", 0);
if(safeactual && safeexpected)
{
csubstr expectedstr = !in_arena ?
parsed_source.sub((size_t)expected[ie].str_start, (size_t)expected[ie].str_len)
:
arena.sub((size_t)expected[ie].str_start, (size_t)expected[ie].str_len);
csubstr actualstr = !in_arena ?
parsed_source.sub((size_t)actual[ia + 1], (size_t)actual[ia + 2])
:
arena.sub((size_t)actual[ia + 1], (size_t)actual[ia + 2]);
_test_eq(expected[ie].scalar, actualstr,
" ref=[{}]~~~{}~~~ vs act=[{}]~~~{}~~~",
expected[ie].scalar.len, expected[ie].scalar,
actualstr.len, actualstr);
if( ! expected[ie].needs_filter)
{
_test_eq(expectedstr, actualstr,
" exp=[{}]~~~{}~~~ vs act=[{}]~~~{}~~~",
expectedstr.len, expectedstr,
actualstr.len, actualstr);
}
}
}
ia += (actual[ia] & ievt::WSTR) ? 3u : 1u;
}
RYML_TRACE_FMT("input:[{}]~~~{}~~~\n"
"parsed:[{}]~~~{}~~~\n",
"arena:[{}]~~~{}~~~\n",
yaml.len, yaml,
parsed_source.len, parsed_source,
arena.len, arena);
EXPECT_TRUE(status);
#undef _test_eq
}
void test_events_ints_invariants(csubstr parsed_yaml,
csubstr arena,
ievt::DataType const* evts,
ievt::DataType evts_sz)
{
char bufpos[100];
char bufprev[100];
EXPECT_GT(evts_sz, 0);
for(ievt::DataType evtpos = 0, evtnumber = 0;
evtpos < evts_sz;
++evtnumber,
evtpos += ((evts[evtpos] & ievt::WSTR) ? 3 : 1))
{
ievt::DataType evt = evts[evtpos];
ievt::DataType prev = {};
ievt::DataType nextpos = evtpos + ((evt & ievt::WSTR) ? 3 : 1);
ievt::DataType next = {};
SCOPED_TRACE(evtpos); // position in the array
SCOPED_TRACE(evtnumber); // event number
SCOPED_TRACE(ievt::to_chars_sub(bufpos, evt));
if(evtpos)
prev = (evt & ievt::PSTR) ? evts[evtpos - 3] : evts[evtpos - 1];
if(nextpos < evts_sz)
next = evts[nextpos];
if(evt & ievt::PSTR)
{
EXPECT_GT(evtnumber, 0);
EXPECT_GE(evtpos, 3);
SCOPED_TRACE(ievt::to_chars_sub(bufprev, prev));
EXPECT_NE(prev & ievt::WSTR, 0);
}
#define _test_str_in_buffer(i) \
do { \
EXPECT_LE(i + 3, evts_sz); \
if (i + 3 <= evts_sz) \
{ \
bool in_arena = evts[i] & ievt::AREN; \
csubstr buf = !in_arena ? parsed_yaml : arena; \
EXPECT_GE(evts[i + 1], 0); \
EXPECT_GE(evts[i + 2], 0); \
if (evts[i + 1] >= 0 && evts[i + 2] >= 0) \
{ \
size_t offset = (size_t)evts[i + 1]; \
size_t len = (size_t)evts[i + 2]; \
EXPECT_LE(offset, buf.len); \
EXPECT_LE(len, buf.len); \
EXPECT_LE(offset + len, buf.len); \
} \
} \
} while(0)
if (evt & ievt::YAML)
{
EXPECT_EQ(evt & ievt::TAGD, 0);
EXPECT_EQ(evt & ievt::TAGV, 0);
EXPECT_EQ(evt & ievt::BSTR, 0);
EXPECT_EQ(evt & ievt::ESTR, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, ievt::YAML);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, ievt::PSTR);
}
if (evt & ievt::TAGD)
{
EXPECT_EQ(evt & ievt::YAML, 0);
EXPECT_EQ(evt & ievt::TAGV, 0);
EXPECT_EQ(evt & ievt::BSTR, 0);
EXPECT_EQ(evt & ievt::ESTR, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, ievt::TAGD);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, ievt::PSTR);
}
if (evt & ievt::TAGV)
{
EXPECT_EQ(evt & ievt::YAML, 0);
EXPECT_EQ(evt & ievt::TAGD, 0);
EXPECT_EQ(evt & ievt::BSTR, 0);
EXPECT_EQ(evt & ievt::ESTR, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, ievt::TAGV);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, ievt::PSTR);
}
if (evt & ievt::BSTR)
{
EXPECT_EQ(evt & ievt::ESTR, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::ESTR)
{
EXPECT_EQ(evt & ievt::BSTR, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::BDOC)
{
EXPECT_EQ(evt & ievt::EDOC, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::EDOC)
{
EXPECT_EQ(evt & ievt::BDOC, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::BSEQ)
{
EXPECT_EQ(evt & ievt::ESEQ, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_NE(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_NE(evt & (ievt::FLOW|ievt::BLCK), ievt::FLOW|ievt::BLCK);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), ievt::KEY_|ievt::VAL_);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::ESEQ)
{
EXPECT_EQ(evt & ievt::BSEQ, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::BMAP)
{
EXPECT_EQ(evt & ievt::EMAP, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_NE(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_NE(evt & (ievt::FLOW|ievt::BLCK), ievt::FLOW|ievt::BLCK);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), ievt::KEY_|ievt::VAL_);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::EMAP)
{
EXPECT_EQ(evt & ievt::BSEQ, 0);
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_EQ(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_EQ(evt & ievt::KEY_, 0);
EXPECT_EQ(evt & ievt::VAL_, 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, 0);
}
if (evt & ievt::SCLR)
{
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_NE(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), ievt::KEY_|ievt::VAL_);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(next & ievt::PSTR, ievt::PSTR);
ievt::DataType style = evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD);
EXPECT_NE(style, 0);
EXPECT_EQ((style & (style << 1)), 0);
_test_str_in_buffer(evtpos);
}
if (evt & ievt::ALIA)
{
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_NE(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), ievt::KEY_|ievt::VAL_);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, ievt::PSTR);
_test_str_in_buffer(evtpos);
}
if (evt & ievt::ANCH)
{
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_NE(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::TAG_, 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), ievt::KEY_|ievt::VAL_);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, ievt::PSTR);
_test_str_in_buffer(evtpos);
}
if (evt & ievt::TAG_)
{
EXPECT_EQ(evt & ievt::EXPL, 0);
EXPECT_NE(evt & ievt::WSTR, 0);
EXPECT_EQ(evt & ievt::SCLR, 0);
EXPECT_EQ(evt & ievt::ALIA, 0);
EXPECT_EQ(evt & ievt::ANCH, 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), 0);
EXPECT_NE(evt & (ievt::KEY_|ievt::VAL_), ievt::KEY_|ievt::VAL_);
EXPECT_EQ(evt & (ievt::BSTR|ievt::ESTR), 0);
EXPECT_EQ(evt & (ievt::BDOC|ievt::EDOC), 0);
EXPECT_EQ(evt & (ievt::BSEQ|ievt::ESEQ), 0);
EXPECT_EQ(evt & (ievt::BMAP|ievt::EMAP), 0);
EXPECT_EQ(evt & (ievt::FLOW|ievt::BLCK), 0);
EXPECT_EQ(evt & (ievt::PLAI|ievt::SQUO|ievt::DQUO|ievt::LITL|ievt::FOLD), 0);
EXPECT_EQ(next & ievt::PSTR, ievt::PSTR);
_test_str_in_buffer(evtpos);
}
#undef _test_str_in_buffer
}
}
} // namespace extra
} // namespace yml
} // namespace c4
// NOLINTEND(hicpp-signed-bitwise)

54
test/test_lib/test_events_ints_helpers.hpp Normal file
View File

@@ -0,0 +1,54 @@
#ifndef _TEST_EVENTS_INTS_HPP_
#define _TEST_EVENTS_INTS_HPP_
#include <c4/yml/extra/event_handler_ints.hpp>
#include <c4/yml/extra/ints_utils.hpp>
#include <c4/yml/extra/event_handler_testsuite.hpp>
#include <gtest/gtest.h>
namespace c4 {
namespace yml {
namespace extra {
// provide a structured input for the event integers, grouping the
// relevant data for the event in a single structure to simplify
// specifying events in tests
struct IntEventWithScalar
{
ievt::DataType flags, str_start, str_len;
csubstr scalar;
bool needs_filter;
IntEventWithScalar(ievt::DataType t, ievt::DataType start=0, ievt::DataType len=0, csubstr sclr={}, bool needs_filter_=false)
: flags(t)
, str_start(start)
, str_len(len)
, scalar(sclr)
, needs_filter(needs_filter_)
{
}
size_t required_size() const { return (flags & ievt::WSTR) ? 3u : 1u; }
};
size_t num_ints(IntEventWithScalar const *evt, size_t evt_size);
void test_events_ints(IntEventWithScalar const* expected, size_t expected_sz,
ievt::DataType const* actual, size_t actual_sz,
csubstr yaml,
csubstr parsed_source,
csubstr arena);
void test_events_ints_invariants(
csubstr parsed_yaml,
csubstr arena,
ievt::DataType const* evts_ints,
ievt::DataType evts_ints_sz);
} // namespace extra
} // namespace yml
} // namespace c4
#endif /* _TEST_EVENTS_INTS_HPP_ */

113
test/test_lib/test_group.cpp
View File

@@ -3,6 +3,10 @@
#endif
#include "test_lib/test_group.hpp"
#include "test_lib/test_case.hpp"
#include "test_lib/test_events_ints_helpers.hpp"
#include <c4/yml/extra/event_handler_ints.hpp>
#include <c4/yml/extra/ints_utils.hpp>
#include <c4/yml/extra/ints_to_testsuite.hpp>
#include <c4/fs/fs.hpp>
#include <fstream>
#include <stdexcept>
@@ -19,13 +23,10 @@ void YmlTestCase::_test_parse_using_ryml(CaseDataLineEndings *cd)
printf("---------------\n%.*s\n---------------\n", (int)c->src.len, c->src.str);
#endif
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
{
auto flags = c->flags;
ExpectError::check_error(&cd->parsed_tree, [this, cd, flags](){
ExpectError::check_error(&cd->parsed_tree, [this, cd]{
parse_in_place(c->fileline, cd->src, &cd->parsed_tree);
if(flags & RESOLVE_REFS)
cd->parsed_tree.resolve();
// if this point was reached, then it means that the expected
// error failed to occur. So print debugging info.
_c4dbg_tree("UNEXPECTED PARSED TREE", cd->parsed_tree);
@@ -44,10 +45,12 @@ void YmlTestCase::_test_parse_using_ryml(CaseDataLineEndings *cd)
}
#endif
if(!(c->flags & EXPECT_RESOLVE_ERROR))
{
SCOPED_TRACE("checking tree invariants of unresolved parsed tree");
test_invariants(cd->parsed_tree);
}
if(!(c->flags & EXPECT_RESOLVE_ERROR))
{
SCOPED_TRACE("checking node invariants of unresolved parsed tree");
test_invariants(cd->parsed_tree.rootref());
@@ -55,6 +58,17 @@ void YmlTestCase::_test_parse_using_ryml(CaseDataLineEndings *cd)
if(c->flags & RESOLVE_REFS)
{
if(c->flags & EXPECT_RESOLVE_ERROR)
{
ExpectError::check_error(&cd->parsed_tree, [&]{
cd->parsed_tree.resolve();
// if this point was reached, then it means that the expected
// error failed to occur. So print debugging info.
_c4dbg_tree("UNEXPECTED RESOLVED TREE", cd->parsed_tree);
}, c->expected_location);
return;
}
cd->parsed_tree.resolve();
_c4dbg_tree("resolved tree!!!", cd->parsed_tree);
{
@@ -95,11 +109,66 @@ void YmlTestCase::_test_parse_using_ryml(CaseDataLineEndings *cd)
}
}
static void _parse_events_ints(csubstr name, substr src, std::vector<int> *ints, std::vector<char> *arena)
{
SCOPED_TRACE("parse_ints");
using I = extra::ievt::DataType;
using Handler = extra::EventHandlerInts;
int estimated_size = extra::estimate_events_ints_size(src);
ints->resize((size_t)estimated_size);
arena->resize(src.len);
Handler handler;
handler.reset(src, to_substr(*arena), ints->data(), (I)ints->size());
ParseEngine<Handler> parser(&handler);
parser.parse_in_place_ev(name, src);
EXPECT_GT(handler.required_size_events(), 0);
ASSERT_GE(estimated_size, handler.required_size_events());
ints->resize((size_t)handler.required_size_events());
ASSERT_TRUE(handler.fits_buffers());
}
void YmlTestCase::_test_parse_using_ints(CaseDataLineEndings *cd)
{
SCOPED_TRACE("test_parse_ints");
#ifdef RYML_DBG
if(_dbg_enabled())
printf("---------------\n%.*s\n---------------\n", (int)c->src.len, c->src.str);
#endif
substr s = to_substr(cd->parse_buf_ints);
auto printints = [&]{
std::cout << extra::events_ints_to_testsuite<std::string>(s, to_csubstr(cd->arena_ints), cd->parsed_ints.data(), (int)cd->parsed_ints.size()) << "\n";
};
if(c->flags & EXPECT_PARSE_ERROR)
{
SCOPED_TRACE("expect error");
ExpectError::check_error(&cd->parsed_tree, [&]{
_parse_events_ints(c->fileline, s, &cd->parsed_ints, &cd->arena_ints);
printints(); // error failed to occur. So print debugging info.
}, c->expected_location);
return;
}
_parse_events_ints(c->fileline, s, &cd->parsed_ints, &cd->arena_ints);
#ifdef RYML_DBG
if(_dbg_enabled())
printints();
#endif
{
SCOPED_TRACE("checking invariants");
extra::test_events_ints_invariants(s, to_csubstr(cd->arena_ints), cd->parsed_ints.data(), (int)cd->parsed_ints.size());
}
}
//-----------------------------------------------------------------------------
void YmlTestCase::_test_emit_yml_stdout(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit(cd);
@@ -112,7 +181,7 @@ void YmlTestCase::_test_emit_json_stdout(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit_json(cd);
@@ -123,7 +192,7 @@ void YmlTestCase::_test_emit_json_stdout(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_emit_yml_cout(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit(cd);
@@ -136,7 +205,7 @@ void YmlTestCase::_test_emit_json_cout(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit_json(cd);
@@ -148,7 +217,7 @@ void YmlTestCase::_test_emit_json_cout(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_emit_yml_stringstream(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit(cd);
@@ -171,7 +240,7 @@ void YmlTestCase::_test_emit_json_stringstream(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit_json(cd);
@@ -192,7 +261,7 @@ void YmlTestCase::_test_emit_json_stringstream(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_emit_yml_ofstream(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit(cd);
@@ -223,7 +292,7 @@ void YmlTestCase::_test_emit_json_ofstream(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit_json(cd);
@@ -252,7 +321,7 @@ void YmlTestCase::_test_emit_json_ofstream(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_emit_yml_string(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit(cd);
@@ -269,7 +338,7 @@ void YmlTestCase::_test_emit_json_string(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit_json(cd);
@@ -284,7 +353,7 @@ void YmlTestCase::_test_emit_json_string(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_emitrs(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
using vtype = std::vector<char>;
@@ -302,7 +371,7 @@ void YmlTestCase::_test_emitrs_json(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
using vtype = std::vector<char>;
@@ -318,7 +387,7 @@ void YmlTestCase::_test_emitrs_json(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_emitrs_cfile(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
auto s = emitrs_yaml<std::string>(cd->parsed_tree);
@@ -337,7 +406,7 @@ void YmlTestCase::_test_emitrs_json_cfile(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
auto s = emitrs_json<std::string>(cd->parsed_tree);
@@ -355,7 +424,7 @@ void YmlTestCase::_test_emitrs_json_cfile(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_complete_round_trip(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS|EXPECT_RESOLVE_ERROR))
return;
_ensure_parse(cd);
_ensure_emit(cd);
@@ -421,7 +490,7 @@ void YmlTestCase::_test_complete_round_trip_json(CaseDataLineEndings *cd)
{
if(!(c->flags & JSON_WRITE))
return;
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
_ensure_parse(cd);
_ensure_emit_json(cd);
@@ -484,7 +553,7 @@ void YmlTestCase::_test_complete_round_trip_json(CaseDataLineEndings *cd)
//-----------------------------------------------------------------------------
void YmlTestCase::_test_recreate_from_ref(CaseDataLineEndings *cd)
{
if(c->flags & EXPECT_PARSE_ERROR)
if(c->flags & (EXPECT_PARSE_ERROR|HAS_CONTAINER_KEYS))
return;
if(cd->parsed_tree.empty())
parse_in_place(c->fileline, cd->src, &cd->parsed_tree);

16
test/test_lib/test_group.def.hpp
View File

@@ -20,6 +20,22 @@ TEST_P(YmlTestCase, parse_windows)
_test_parse_using_ryml(&d->windows_style);
}
//-----------------------------------------------------------------------------
TEST_P(YmlTestCase, parse_ints_unix)
{
SCOPED_TRACE("unix style\n" + c->filelinebuf + ": case");
_test_parse_using_ints(&d->unix_style);
}
TEST_P(YmlTestCase, parse_ints_windows)
{
SCOPED_TRACE("windows style\n" + c->filelinebuf + ": case");
_test_parse_using_ints(&d->windows_style);
}
//-----------------------------------------------------------------------------
TEST_P(YmlTestCase, emit_yml_unix_stdout)
{

1
test/test_lib/test_group.hpp
View File

@@ -71,6 +71,7 @@ struct YmlTestCase : public ::testing::TestWithParam<csubstr>
}
void _test_parse_using_ryml(CaseDataLineEndings *cd);
void _test_parse_using_ints(CaseDataLineEndings *cd);
void _test_emit_yml_stdout(CaseDataLineEndings *cd);
void _test_emit_json_stdout(CaseDataLineEndings *cd);

48
test/test_location.cpp
View File

@@ -46,7 +46,7 @@ TEST(locations, error_is_triggered_querying_with_locations_disabled)
Parser parser(&evt_handler, ParserOptions().locations(false));
Tree t = parse_in_arena(&parser, "test", "foo: bar");
parsed_ok = true;
(void)parser.location(t["foo"]);
t["foo"].location(parser);
});
EXPECT_TRUE(parsed_ok);
}
@@ -55,7 +55,7 @@ TEST(locations, error_is_triggered_querying_with_locations_disabled)
#define _checkloc(node, line_, col_, str) \
{ \
const Location loc = parser.location(node); \
const Location loc = node.location(parser); \
EXPECT_EQ(loc.name, "myfile.yml"); \
EXPECT_EQ(loc.line, line_); \
EXPECT_EQ(loc.col, col_); \
@@ -661,7 +661,7 @@ TEST(locations, issue260_0)
Line One
Line Two)");
EXPECT_EQ(tree["Body"][2]["Name"].val(), "Dog");
EXPECT_EQ(parser.location(tree["Body"][2]["Name"]).line, 12u);
EXPECT_EQ(tree["Body"][2]["Name"].location(parser).line, 12u);
}
TEST(locations, issue260_1)
@@ -702,27 +702,27 @@ TEST(locations, issue260_1)
Script: |
Line One
Line Two)");
EXPECT_EQ(parser.location(tree["Body"][0]).line, 1u);
EXPECT_EQ(parser.location(tree["Body"][1]).line, 3u);
EXPECT_EQ(parser.location(tree["Body"][2]).line, 8u);
EXPECT_EQ(parser.location(tree["Body"][3]).line, 13u);
EXPECT_EQ(parser.location(tree["Body"][4]).line, 18u);
EXPECT_EQ(parser.location(tree["Body"][5]).line, 23u);
EXPECT_EQ(parser.location(tree["Body"][6]).line, 28u);
EXPECT_EQ(parser.location(tree["Body"][0]["Id"]).line, 1u);
EXPECT_EQ(parser.location(tree["Body"][1]["Id"]).line, 3u);
EXPECT_EQ(parser.location(tree["Body"][2]["Id"]).line, 8u);
EXPECT_EQ(parser.location(tree["Body"][3]["Id"]).line, 13u);
EXPECT_EQ(parser.location(tree["Body"][4]["Id"]).line, 18u);
EXPECT_EQ(parser.location(tree["Body"][5]["Id"]).line, 23u);
EXPECT_EQ(parser.location(tree["Body"][6]["Id"]).line, 28u);
EXPECT_EQ(parser.location(tree["Body"][0]["Name"]).line, 1u+1u);
EXPECT_EQ(parser.location(tree["Body"][1]["Name"]).line, 3u+1u);
EXPECT_EQ(parser.location(tree["Body"][2]["Name"]).line, 8u+1u);
EXPECT_EQ(parser.location(tree["Body"][3]["Name"]).line, 13u+1u);
EXPECT_EQ(parser.location(tree["Body"][4]["Name"]).line, 18u+1u);
EXPECT_EQ(parser.location(tree["Body"][5]["Name"]).line, 23u+1u);
EXPECT_EQ(parser.location(tree["Body"][6]["Name"]).line, 28u+1u);
EXPECT_EQ(tree["Body"][0].location(parser).line, 1u);
EXPECT_EQ(tree["Body"][1].location(parser).line, 3u);
EXPECT_EQ(tree["Body"][2].location(parser).line, 8u);
EXPECT_EQ(tree["Body"][3].location(parser).line, 13u);
EXPECT_EQ(tree["Body"][4].location(parser).line, 18u);
EXPECT_EQ(tree["Body"][5].location(parser).line, 23u);
EXPECT_EQ(tree["Body"][6].location(parser).line, 28u);
EXPECT_EQ(tree["Body"][0]["Id"].location(parser).line, 1u);
EXPECT_EQ(tree["Body"][1]["Id"].location(parser).line, 3u);
EXPECT_EQ(tree["Body"][2]["Id"].location(parser).line, 8u);
EXPECT_EQ(tree["Body"][3]["Id"].location(parser).line, 13u);
EXPECT_EQ(tree["Body"][4]["Id"].location(parser).line, 18u);
EXPECT_EQ(tree["Body"][5]["Id"].location(parser).line, 23u);
EXPECT_EQ(tree["Body"][6]["Id"].location(parser).line, 28u);
EXPECT_EQ(tree["Body"][0]["Name"].location(parser).line, 1u+1u);
EXPECT_EQ(tree["Body"][1]["Name"].location(parser).line, 3u+1u);
EXPECT_EQ(tree["Body"][2]["Name"].location(parser).line, 8u+1u);
EXPECT_EQ(tree["Body"][3]["Name"].location(parser).line, 13u+1u);
EXPECT_EQ(tree["Body"][4]["Name"].location(parser).line, 18u+1u);
EXPECT_EQ(tree["Body"][5]["Name"].location(parser).line, 23u+1u);
EXPECT_EQ(tree["Body"][6]["Name"].location(parser).line, 28u+1u);
}

0
test/test_scalar_empty.cpp
View File

48
test/test_scalar_null.cpp
View File

@@ -305,18 +305,18 @@ block:
//
// check also locations because nullity may influence the search
// for location
EXPECT_EQ(parser.location(fdquoted).line, 0u);
EXPECT_EQ(parser.location(bdquoted).line, 2u);
EXPECT_EQ(parser.location(fsquoted).line, 0u);
EXPECT_EQ(parser.location(bsquoted).line, 3u);
EXPECT_EQ(parser.location(bliteral).line, 4u);
EXPECT_EQ(parser.location(bfolded ).line, 5u);
EXPECT_EQ(parser.location(fplain ).line, 0u);
EXPECT_EQ(parser.location(bplain ).line, 6u);
EXPECT_EQ(parser.location(fenull ).line, 0u);
EXPECT_EQ(parser.location(benull ).line, 7u);
EXPECT_EQ(parser.location(ftilde ).line, 0u);
EXPECT_EQ(parser.location(btilde ).line, 8u);
EXPECT_EQ(fdquoted.location(parser).line, 0u);
EXPECT_EQ(bdquoted.location(parser).line, 2u);
EXPECT_EQ(fsquoted.location(parser).line, 0u);
EXPECT_EQ(bsquoted.location(parser).line, 3u);
EXPECT_EQ(bliteral.location(parser).line, 4u);
EXPECT_EQ(bfolded .location(parser).line, 5u);
EXPECT_EQ(fplain .location(parser).line, 0u);
EXPECT_EQ(bplain .location(parser).line, 6u);
EXPECT_EQ(fenull .location(parser).line, 0u);
EXPECT_EQ(benull .location(parser).line, 7u);
EXPECT_EQ(ftilde .location(parser).line, 0u);
EXPECT_EQ(btilde .location(parser).line, 8u);
//
EXPECT_TRUE(fdquoted.has_val());
EXPECT_TRUE(bdquoted.has_val());
@@ -460,18 +460,18 @@ block:
//
// check also locations because nullity may influence the search
// for location
EXPECT_EQ(parser.location(fdquoted).line, 0u);
EXPECT_EQ(parser.location(bdquoted).line, 2u);
EXPECT_EQ(parser.location(fsquoted).line, 0u);
EXPECT_EQ(parser.location(bsquoted).line, 3u);
EXPECT_EQ(parser.location(bliteral).line, 5u);
EXPECT_EQ(parser.location(bfolded ).line, 7u);
EXPECT_EQ(parser.location(fplain ).line, 0u);
EXPECT_EQ(parser.location(bplain ).line, 8u);
EXPECT_EQ(parser.location(fenull ).line, 0u);
EXPECT_EQ(parser.location(benull ).line, 9u);
EXPECT_EQ(parser.location(ftilde ).line, 0u);
EXPECT_EQ(parser.location(btilde ).line, 10u);
EXPECT_EQ(fdquoted.location(parser).line, 0u);
EXPECT_EQ(bdquoted.location(parser).line, 2u);
EXPECT_EQ(fsquoted.location(parser).line, 0u);
EXPECT_EQ(bsquoted.location(parser).line, 3u);
EXPECT_EQ(bliteral.location(parser).line, 5u);
EXPECT_EQ(bfolded .location(parser).line, 7u);
EXPECT_EQ(fplain .location(parser).line, 0u);
EXPECT_EQ(bplain .location(parser).line, 8u);
EXPECT_EQ(fenull .location(parser).line, 0u);
EXPECT_EQ(benull .location(parser).line, 9u);
EXPECT_EQ(ftilde .location(parser).line, 0u);
EXPECT_EQ(btilde .location(parser).line, 10u);
//
EXPECT_TRUE(fdquoted.has_key());
EXPECT_TRUE(bdquoted.has_key());

7
test/test_singleheader/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ c4_project(VERSION 0.9.0
AUTHOR "Joao Paulo Magalhaes <dev@jpmag.me>")
# amalgamate ryml to get the single header
function(amalgamate_ryml header_dir header_file)
function(amalgamate_ryml header_dir header_file basename)
set(rymldir "${CMAKE_CURRENT_LIST_DIR}/../..")
set(singleheaderdir "${rymldir}/src_singleheader")
set(singleheader "${singleheaderdir}/ryml_all.hpp")
@@ -17,13 +17,12 @@ function(amalgamate_ryml header_dir header_file)
LIST_DIRECTORIES FALSE
CONFIGURE_DEPENDS "${rymldir}/src")
add_custom_command(OUTPUT "${singleheader}"
COMMAND python "${amscript}" "${singleheader}"
COMMENT "python ${amscript} ${singleheader}"
COMMAND python "${amscript}" --events tree -- "${singleheader}"
DEPENDS ${srcfiles} "${amscript}" "${rymldir}/ext/c4core/cmake/amalgamate_utils.py")
set(${header_dir} "${singleheaderdir}" PARENT_SCOPE)
set(${header_file} "${singleheader}" PARENT_SCOPE)
endfunction()
amalgamate_ryml(header_dir header_file)
amalgamate_ryml(header_dir header_file ryml_all.hpp)
c4_add_library(ryml
INC_DIRS

187
test/test_suite.cpp → test/testsuite.cpp
View File

@@ -7,10 +7,14 @@
#include <c4/yml/detail/checks.hpp>
#endif
#include "test_lib/test_case.hpp"
#include "test_suite/test_suite_common.hpp"
#include "test_suite/test_suite_parts.hpp"
#include "test_suite/test_suite_events.hpp"
#include "test_suite/test_suite_event_handler.hpp"
#include "test_lib/test_engine.hpp"
#include "test_lib/test_events_ints_helpers.hpp"
#include "testsuite/testsuite_common.hpp"
#include "testsuite/testsuite_parts.hpp"
#include "testsuite/testsuite_events.hpp"
#include "c4/yml/extra/event_handler_testsuite.hpp"
#include "c4/yml/extra/event_handler_ints.hpp"
#include "c4/yml/extra/ints_to_testsuite.hpp"
#include <c4/fs/fs.hpp>
#include <c4/log/log.hpp>
#include <gtest/gtest.h>
@@ -117,12 +121,14 @@ struct TestSuiteCaseEvents
mutable Tree tmp_tree_from_emitted_events = {};
std::string tmp_events_emitted_from_parsed_tree = {};
void compare_emitted_events_to_reference_events(std::string const& emitted_events, bool ignore_container_style, bool ignore_scalar_style)
void compare_events(csubstr emitted_events, bool ignore_container_style, bool ignore_scalar_style)
{
if(compare_events(to_csubstr(reference_events), to_csubstr(emitted_events), ignore_container_style, ignore_scalar_style))
{
EXPECT_EQ(emitted_events, reference_events);
}
test_compare_events(to_csubstr(reference_events),
emitted_events,
/*ignore_doc_style*/true,
ignore_container_style,
ignore_scalar_style,
/*ignore_tag*/true);
}
};
@@ -136,9 +142,12 @@ struct TestSequenceLevel
size_t level;
TestSequenceLevel *prev;
csubstr filename;
std::string src_orig;
std::string src_tree;
std::string src_tree_json;
std::string src_evts;
std::string src_evts_ints;
std::string arena_evts_ints;
EventHandlerTree evt_handler_tree;
EventHandlerTree evt_handler_tree_json;
Parser parser_tree;
@@ -148,9 +157,14 @@ struct TestSequenceLevel
std::string emitted_from_tree_parsed_from_src;
std::string emitted_from_tree_parsed_from_src_json;
EventHandlerYamlStd::EventSink evt_str_sink;
EventHandlerYamlStd evt_handler_str_sink;
ParseEngine<EventHandlerYamlStd> parser_str_sink;
extra::EventHandlerTestSuite::EventSink evt_str_sink;
extra::EventHandlerTestSuite evt_handler_str;
ParseEngine<extra::EventHandlerTestSuite> parser_str;
extra::EventHandlerInts evt_handler_ints;
ParseEngine<extra::EventHandlerInts> parser_ints;
std::vector<extra::EventHandlerInts::value_type> buffer_ints;
std::string evts_test_suite_from_ints;
bool immutable = false;
bool reuse = false;
@@ -159,6 +173,7 @@ struct TestSequenceLevel
bool tree_was_emitted = false;
bool tree_was_emitted_json = false;
bool events_were_generated = false;
bool events_ints_were_generated = false;
TestSequenceLevel()
: evt_handler_tree()
@@ -166,8 +181,11 @@ struct TestSequenceLevel
, parser_tree(&evt_handler_tree)
, parser_tree_json(&evt_handler_tree_json)
, evt_str_sink()
, evt_handler_str_sink(&evt_str_sink)
, parser_str_sink(&evt_handler_str_sink)
, evt_handler_str(&evt_str_sink)
, parser_str(&evt_handler_str)
, evt_handler_ints()
, parser_ints(&evt_handler_ints)
, buffer_ints()
{
}
@@ -177,7 +195,9 @@ struct TestSequenceLevel
prev = prev_;
filename = filename_;
src_tree.assign(src_.begin(), src_.end());
src_orig = src_tree;
src_evts = src_tree;
src_evts_ints = src_tree;
immutable = immutable_;
reuse = reuse_;
tree_was_parsed = false;
@@ -185,6 +205,7 @@ struct TestSequenceLevel
tree_was_emitted = false;
tree_was_emitted_json = false;
events_were_generated = false;
events_ints_were_generated = false;
}
void receive_src(TestSequenceLevel & prev_)
@@ -200,8 +221,10 @@ struct TestSequenceLevel
tree_was_parsed = false;
tree_was_emitted = false;
events_were_generated = false;
events_ints_were_generated = false;
src_tree = prev_.emitted_from_tree_parsed_from_src;
src_evts = src_tree;
src_evts_ints = src_tree;
}
}
@@ -298,15 +321,52 @@ struct TestSequenceLevel
return;
if(prev)
receive_src(*prev);
_nfo_logf("level[{}]: parsing source:\n{}", level, src_evts);
_nfo_logf("level[{}]: parsing source to events:\n{}", level, src_evts);
evt_str_sink.clear();
evt_handler_str_sink.reset();
evt_handler_str_sink.m_stack.m_callbacks = get_callbacks();
parser_str_sink.parse_in_place_ev(filename, to_substr(src_evts));
evt_handler_str.reset();
evt_handler_str.m_stack.m_callbacks = get_callbacks();
parser_str.parse_in_place_ev(filename, to_substr(src_evts));
EXPECT_NE(evt_str_sink.size(), 0);
events_were_generated = true;
}
void parse_yaml_to_events_ints()
{
using I = extra::ievt::DataType;
if(events_ints_were_generated)
return;
if(prev)
receive_src(*prev);
_nfo_logf("level[{}]: parsing source to ints:\n{}", level, src_evts_ints);
buffer_ints.resize(32);
int size_estimated = extra::estimate_events_ints_size(to_csubstr(src_evts_ints));
evt_handler_ints.m_stack.m_callbacks = get_callbacks();
evt_handler_ints.reset(to_substr(src_evts_ints), to_substr(arena_evts_ints), buffer_ints.data(), (I)buffer_ints.size());
parser_ints.parse_in_place_ev(filename, to_substr(src_evts_ints));
EXPECT_GE(size_estimated, evt_handler_ints.required_size_events());
size_t sz = (size_t)evt_handler_ints.required_size_events();
if (!evt_handler_ints.fits_buffers())
{
buffer_ints.resize(sz);
arena_evts_ints.resize(evt_handler_ints.required_size_arena());
src_evts_ints = src_orig;
evt_handler_ints.reset(to_substr(src_evts_ints), to_substr(arena_evts_ints), buffer_ints.data(), (I)buffer_ints.size());
parser_ints.parse_in_place_ev(filename, to_substr(src_evts_ints));
size_t sz2 = (size_t)evt_handler_ints.required_size_events();
ASSERT_EQ(sz2, sz);
sz = sz2;
}
ASSERT_LE(sz, buffer_ints.size());
buffer_ints.resize(sz);
#ifdef RYML_DBG
extra::events_ints_print(to_csubstr(src_evts_ints), to_substr(arena_evts_ints), buffer_ints.data(), (I)sz);
#endif
extra::test_events_ints_invariants(to_csubstr(src_evts_ints), to_substr(arena_evts_ints), buffer_ints.data(), (I)sz);
EXPECT_GT(evt_handler_ints.required_size_events(), 0);
extra::events_ints_to_testsuite(to_csubstr(src_evts_ints), to_substr(arena_evts_ints), buffer_ints.data(), (I)buffer_ints.size(), &evts_test_suite_from_ints);
events_ints_were_generated = true;
}
void emit_parsed_tree()
{
if(tree_was_emitted)
@@ -543,6 +603,28 @@ struct TestSequenceData
}
}
void parse_yaml_to_events_ints(size_t num)
{
SKIP_IF(allowed_failure);
//SKIP_IF(has_container_keys); // DO IT!
for(size_t i = 0; i < num; ++i)
{
if(!expect_error)
{
levels[i].parse_yaml_to_events_ints();
if(has_container_keys)
break;
}
else
{
ExpectError::check_error([&]{
levels[i].parse_yaml_to_events_ints();
});
break; // because we expect error,we cannot go on to the next
}
}
}
void emit_tree_parsed_from_src(size_t num)
{
SKIP_IF(allowed_failure);
@@ -655,15 +737,33 @@ struct TestSequenceData
for(size_t i = 0; i < num; ++i)
{
levels[i].parse_yaml_to_events();
csubstr result = levels[i].evt_str_sink;
events->compare_emitted_events_to_reference_events(std::string(result.str, result.len),
/*ignore_container_style*/false,
/*ignore_scalar_style*/(num>0));
events->compare_events(levels[i].evt_str_sink,
/*ignore_container_style*/false,
/*ignore_scalar_style*/(num>0));
}
}
void compare_events_str(size_t num, TestSuiteCaseEvents *events)
{
ASSERT_EQ(num, 1); // FIXME
SKIP_IF(allowed_failure);
parse_yaml_to_events(1);
events->compare_events(levels[0].evt_str_sink,
/*ignore_container_style*/false,
/*ignore_scalar_style*/(num>0));
}
void compare_events_ints_str(size_t num, TestSuiteCaseEvents *events)
{
ASSERT_EQ(num, 1); // FIXME
SKIP_IF(allowed_failure);
parse_yaml_to_events_ints(1);
events->compare_events(to_csubstr(levels[0].evts_test_suite_from_ints),
/*ignore_container_style*/false,
/*ignore_scalar_style*/false);
}
bool m_expected_error_to_tree_checked = false;
bool m_expected_error_to_events_checked = false;
bool m_expected_error_to_events_ints_checked = false;
void check_expected_error()
{
SKIP_IF(allowed_failure);
@@ -686,6 +786,17 @@ struct TestSequenceData
});
m_expected_error_to_events_checked = true;
}
void check_expected_error_events_ints()
{
SKIP_IF(allowed_failure);
//SKIP_IF(has_container_keys); // DO IT!
if(m_expected_error_to_events_ints_checked)
return;
ExpectError::check_error([this]{
levels[0].parse_yaml_to_events_ints();
});
m_expected_error_to_events_ints_checked = true;
}
};
@@ -820,6 +931,38 @@ TEST(which##_errors, check_expected_error_src_to_events) \
{ \
SKIP_IF(!g_suite_case->test_case_expects_error); \
g_suite_case->which.check_expected_error_events(); \
} \
\
TEST(which##_errors, check_expected_error_src_to_events_ints) \
{ \
SKIP_IF(!g_suite_case->test_case_expects_error); \
g_suite_case->which.check_expected_error_events_ints(); \
} \
\
\
/*-----------------------------------------------*/ \
\
TEST(which##_events_from_src, parse_yaml_to_events) \
{ \
g_suite_case->which.parse_yaml_to_events(1); \
} \
\
TEST(which##_events_from_src, compare_events_to_ref_events) \
{ \
g_suite_case->which.compare_events_ints_str(1, &g_suite_case->events); \
} \
\
\
/*-----------------------------------------------*/ \
\
TEST(which##_events_ints_from_src, parse_yaml_to_events_ints) \
{ \
g_suite_case->which.parse_yaml_to_events_ints(1); \
} \
\
TEST(which##_events_ints_from_src, compare_events_ints_to_ref_events) \
{ \
g_suite_case->which.compare_events_ints_str(1, &g_suite_case->events); \
} \
\
\

0
test/test_suite/test_suite_common.hpp → test/testsuite/testsuite_common.hpp
View File

149
test/test_suite/test_suite_events.cpp → test/testsuite/testsuite_events.cpp
View File

@@ -1,8 +1,11 @@
#include "test_suite_events.hpp"
#include "test_suite_event_handler.hpp"
#include "test_suite_common.hpp"
#ifndef RYML_SINGLE_HEADER
#include "testsuite_events.hpp"
#include "testsuite_common.hpp"
#include "test_lib/test_engine.hpp"
#ifdef RYML_SINGLE_HEADER
#include <ryml_all.hpp>
#else
#include <c4/yml/detail/stack.hpp>
#include "c4/yml/extra/event_handler_testsuite.hpp"
#endif
namespace c4 {
@@ -11,9 +14,9 @@ namespace yml {
std::string emit_events_from_source(substr src)
{
EventHandlerYamlStd::EventSink sink;
EventHandlerYamlStd handler(&sink);
ParseEngine<EventHandlerYamlStd> parser(&handler);
extra::EventHandlerTestSuite::EventSink sink;
extra::EventHandlerTestSuite handler(&sink);
ParseEngine<extra::EventHandlerTestSuite> parser(&handler);
parser.parse_in_place_ev("(testyaml)", src);
csubstr result = sink;
return std::string(result.str, result.len);
@@ -168,138 +171,6 @@ struct Scalar
} // namespace /*anon*/
csubstr parse_anchor_and_tag(csubstr tokens, OptionalScalar *anchor, OptionalScalar *tag)
{
*anchor = OptionalScalar{};
*tag = OptionalScalar{};
if(tokens.begins_with('&'))
{
size_t pos = tokens.first_of(' ');
if(pos == (size_t)csubstr::npos)
{
*anchor = tokens.sub(1);
tokens = {};
}
else
{
*anchor = tokens.first(pos).sub(1);
tokens = tokens.right_of(pos);
}
_nfo_logf("anchor: {}", anchor->get());
}
if(tokens.begins_with('<'))
{
size_t pos = tokens.find('>');
RYML_ASSERT(pos != (size_t)csubstr::npos);
*tag = tokens.first(pos + 1);
tokens = tokens.right_of(pos).triml(' ');
_nfo_logf("tag: {}", tag->maybe_get());
}
return tokens;
}
bool compare_events(csubstr ref_evts, csubstr emt_evts, bool ignore_container_style=false, bool ignore_scalar_style=false)
{
auto diff_line_with_optional_ending = [](csubstr ref, csubstr emt, csubstr optional_ending){
RYML_ASSERT(ref != emt);
ref = ref.stripr(optional_ending).trimr(' ');
emt = emt.stripr(optional_ending).trimr(' ');
bool diff = ref != emt;
return diff;
};
auto diff_val_with_scalar_wildcard = [](csubstr ref, csubstr emt){
RYML_ASSERT(ref.begins_with("=VAL "));
RYML_ASSERT(emt.begins_with("=VAL "));
ref = ref.sub(5);
emt = emt.sub(5);
OptionalScalar reftag = {}, refanchor = {};
OptionalScalar emttag = {}, emtanchor = {};
if((bool(reftag) != bool(emttag)) || (reftag && (reftag.get() != emttag.get())))
return true;
if((bool(refanchor) != bool(emtanchor)) || (refanchor && (refanchor.get() != emtanchor.get())))
return true;
ref = parse_anchor_and_tag(ref, &refanchor, &reftag).triml(' ');
emt = parse_anchor_and_tag(emt, &emtanchor, &emttag).triml(' ');
RYML_ASSERT(ref.len > 0);
RYML_ASSERT(emt.len > 0);
RYML_ASSERT(ref[0] == ':' || ref[0] == '\'' || ref[0] == '"' || ref[0] == '|' || ref[0] == '>');
RYML_ASSERT(emt[0] == ':' || emt[0] == '\'' || emt[0] == '"' || emt[0] == '|' || emt[0] == '>');
ref = ref.sub(1);
emt = emt.sub(1);
if(ref != emt)
return true;
return false;
};
if(bool(ref_evts.len) != bool(emt_evts.len))
return true;
size_t posref = 0;
size_t posemt = 0;
bool fail = false;
while(posref < ref_evts.len && posemt < emt_evts.len)
{
const size_t endref = ref_evts.find('\n', posref);
const size_t endemt = emt_evts.find('\n', posemt);
if((endref == npos || endemt == npos) && (endref != endemt))
{
fail = true;
break;
}
csubstr ref = ref_evts.range(posref, endref);
csubstr emt = emt_evts.range(posemt, endemt);
if(ref != emt)
{
if(ref.begins_with("+DOC"))
{
if(diff_line_with_optional_ending(ref, emt, "---"))
{
fail = true;
break;
}
}
else if(ref.begins_with("-DOC"))
{
if(diff_line_with_optional_ending(ref, emt, "..."))
{
fail = true;
break;
}
}
else if(ignore_container_style && ref.begins_with("+MAP"))
{
if(diff_line_with_optional_ending(ref, emt, "{}"))
{
fail = true;
break;
}
}
else if(ignore_container_style && ref.begins_with("+SEQ"))
{
if(diff_line_with_optional_ending(ref, emt, "[]"))
{
fail = true;
break;
}
}
else if(ignore_scalar_style && ref.begins_with("=VAL"))
{
if(diff_val_with_scalar_wildcard(ref, emt))
{
fail = true;
break;
}
}
else
{
fail = true;
break;
}
}
posref = endref + 1u;
posemt = endemt + 1u;
}
return fail;
}
void parse_events_to_tree(csubstr src, Tree *C4_RESTRICT tree_)
{
struct ParseLevel { id_type tree_node; };

15
test/test_suite/test_suite_events.hpp → test/testsuite/testsuite_events.hpp
View File

@@ -37,21 +37,6 @@ CharContainer emit_events_from_tree(Tree const& C4_RESTRICT tree)
return result;
}
struct OptionalScalar
{
csubstr val = {};
bool was_set = false;
inline operator csubstr() const { return get(); }
inline operator bool() const { return was_set; }
void operator= (csubstr v) { val = v; was_set = true; }
csubstr get() const { RYML_ASSERT(was_set); return val; }
csubstr maybe_get() const { return was_set ? val : csubstr(""); }
};
csubstr parse_anchor_and_tag(csubstr tokens, OptionalScalar *anchor, OptionalScalar *tag);
bool compare_events(csubstr ref_evts, csubstr emt_evts, bool ignore_container_style, bool ignore_scalar_style);
} // namespace yml
} // namespace c4

2
test/test_suite/test_suite_events_emitter.cpp → test/testsuite/testsuite_events_emitter.cpp
View File

@@ -1,7 +1,7 @@
#ifndef RYML_SINGLE_HEADER
#include <c4/yml/std/string.hpp>
#endif
#include "test_suite_events.hpp"
#include "testsuite_events.hpp"
namespace c4 {
namespace yml {

8
test/test_suite/test_suite_parts.cpp → test/testsuite/testsuite_parts.cpp
View File

@@ -1,4 +1,4 @@
#include "./test_suite_parts.hpp"
#include "./testsuite_parts.hpp"
namespace c4 {
namespace yml {
@@ -19,6 +19,9 @@ constexpr const AllowedFailure allowed_failures[] = {
//
// These tests are temporarily skipped, and cover issues that must be fixed.
// this is a container key that is parsed incorrectly
_("M2N8_01-in_yaml" , "container key parsed incorrectly (events)"),
// errors that fail to materialize
_("3HFZ-error" , "should not accept scalar after ..."),
_("4EJS-error" , "should not accept tabs as indendation in a mapping"),
@@ -134,8 +137,7 @@ constexpr const AllowedFailure container_key_cases[] = {
// these cases have container keys, and cannot be parsed into the
// ryml tree. However, they CAN be parsed by the ryml parse engine.
// Therefore the tests are enabled if they only use the YAML event
// emitter based on the parse engine.
// Therefore they are treated specifically.
_("4FJ6-in_yaml" , "only scalar keys allowed (keys cannot be containers)"),
_("4FJ6-out_yaml" , "only scalar keys allowed (keys cannot be containers)"),

0
test/test_suite/test_suite_parts.hpp → test/testsuite/testsuite_parts.hpp
View File

19
tools/CMakeLists.txt
View File

@@ -6,10 +6,19 @@ c4_add_executable(ryml-parse-emit
c4_add_executable(ryml-yaml-events
SOURCES yaml_events.cpp
../test/test_suite/test_suite_events.hpp
../test/test_suite/test_suite_events_emitter.cpp
../test/test_suite/test_suite_event_handler.hpp
../test/test_suite/test_suite_event_handler.cpp
INC_DIRS ../test
../src_extra/c4/yml/extra/event_handler_ints.hpp
../src_extra/c4/yml/extra/event_handler_ints.cpp
../src_extra/c4/yml/extra/event_handler_testsuite.hpp
../src_extra/c4/yml/extra/event_handler_testsuite.cpp
../src_extra/c4/yml/extra/ints_utils.hpp
../src_extra/c4/yml/extra/ints_utils.cpp
../src_extra/c4/yml/extra/ints_to_testsuite.hpp
../src_extra/c4/yml/extra/ints_to_testsuite.cpp
../src_extra/c4/yml/extra/scalar.hpp
../src_extra/c4/yml/extra/scalar.cpp
../src_extra/c4/yml/extra/string.hpp
../test/testsuite/testsuite_events.hpp
../test/testsuite/testsuite_events_emitter.cpp
INC_DIRS ../test ../src_extra
LIBS ryml c4fs
FOLDER tools)

117
tools/amalgamate.py
View File

@@ -4,7 +4,8 @@ from os.path import abspath, dirname
import sys
import subprocess
import argparse
from typing import List
from enum import Enum
projdir = abspath(dirname(dirname(__file__)))
sys.path.insert(0, f"{projdir}/ext/c4core/cmake")
@@ -12,6 +13,33 @@ import amalgamate_utils as am
sys.path.insert(0, f"{projdir}/ext/c4core/tools")
import amalgamate as am_c4core
class Event(Enum):
tree = "tree"
testsuite = "testsuite"
ints = "ints"
ints_utils = "ints_utils"
ints_to_testsuite = "ints_to_testsuite"
all = "all"
none = "none"
def __str__(self):
return self.value
event_doc = {
Event.tree: """(the default) enable the normal ryml event handler
to create the tree, and additionally the Tree, Node, parser and
emitter utilities; if this is not enabled, none of these
components will be included in the amalgamated file""",
Event.testsuite: "enable the (extra) YAML test suite event handler",
Event.ints: "enable the (extra) integer-based event handler",
Event.ints_utils: "enable the (extra) integer-based event handler utils",
Event.ints_to_testsuite: "enable the (extra) integer events conversion to testsuite events",
Event.all: "enable all event handlers",
Event.none: "disable all event handlers",
}
ryml_defmacro = "RYML_SINGLE_HDR_DEFINE_NOW"
c4core_defmacro = "C4CORE_SINGLE_HDR_DEFINE_NOW"
exports_def_code = f""" // shared library: export when defining
@@ -42,7 +70,8 @@ c4core_def_code = f""" // propagate defines to c4core
def amalgamate_ryml(filename: str,
with_c4core: bool,
with_fastfloat: bool,
with_stl: bool):
with_stl: bool,
events: List[Event]):
c4core_amalgamated = ""
if with_c4core:
c4core_amalgamated = "src/c4/c4core_all.hpp"
@@ -66,11 +95,11 @@ INSTRUCTIONS:
`_RYML_SINGLE_HEADER_AMALGAMATED_HPP_`, ie like this:
```
#ifndef _RYML_SINGLE_HEADER_AMALGAMATED_HPP_
#include <ryml_all.hpp>
#include <header_file_name.hpp>
#endif
```
- In one (and only one) of your project source files, #define
- In one -- and only one -- of your project source files, #define
{ryml_defmacro} and then include this header. This will enable
the function and class definitions in the header file.
@@ -79,6 +108,13 @@ INSTRUCTIONS:
symbol export/import.
"""
def has_evt(*which):
if Event.all in events:
return True
for e in which:
if e in events:
return True
return False
srcfiles = [
am.cmttext(ryml_preamble),
am.cmtfile("LICENSE.txt"),
@@ -91,40 +127,50 @@ INSTRUCTIONS:
"src/c4/yml/common.hpp",
"src/c4/yml/node_type.hpp",
"src/c4/yml/tag.hpp",
"src/c4/yml/tree.hpp",
"src/c4/yml/node.hpp",
"src/c4/yml/writer.hpp",
"src/c4/yml/detail/parser_dbg.hpp",
am.injcode("#define C4_YML_EMIT_DEF_HPP_"),
"src/c4/yml/emit.hpp",
"src/c4/yml/emit.def.hpp",
am.onlyif(has_evt(Event.tree), "src/c4/yml/tree.hpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/node.hpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/writer.hpp"),
"src/c4/yml/detail/dbgprint.hpp",
am.onlyif(has_evt(Event.tree), am.injcode("#define C4_YML_EMIT_DEF_HPP_")),
am.onlyif(has_evt(Event.tree), "src/c4/yml/emit.hpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/emit.def.hpp"),
"src/c4/yml/detail/stack.hpp",
"src/c4/yml/filter_processor.hpp",
"src/c4/yml/parser_state.hpp",
"src/c4/yml/event_handler_stack.hpp",
"src/c4/yml/event_handler_tree.hpp",
am.onlyif(has_evt(Event.tree), "src/c4/yml/event_handler_tree.hpp"),
am.onlyif(has_evt(Event.ints), "src_extra/c4/yml/extra/event_handler_ints.hpp"),
am.onlyif(has_evt(Event.ints_utils), "src_extra/c4/yml/extra/ints_utils.hpp"),
am.onlyif(has_evt(Event.ints_to_testsuite), "src_extra/c4/yml/extra/ints_to_testsuite.hpp"),
am.onlyif(has_evt(Event.testsuite), "src_extra/c4/yml/extra/string.hpp"),
am.onlyif(has_evt(Event.testsuite), "src_extra/c4/yml/extra/event_handler_testsuite.hpp"),
am.onlyif(has_evt(Event.ints_utils, Event.testsuite), "src_extra/c4/yml/extra/scalar.hpp"),
"src/c4/yml/parse_engine.hpp",
"src/c4/yml/preprocess.hpp",
"src/c4/yml/reference_resolver.hpp",
"src/c4/yml/parse.hpp",
am.onlyif(with_stl, "src/c4/yml/std/map.hpp"),
am.onlyif(with_stl, "src/c4/yml/std/string.hpp"),
am.onlyif(with_stl, "src/c4/yml/std/vector.hpp"),
am.onlyif(with_stl, "src/c4/yml/std/std.hpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/reference_resolver.hpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/parse.hpp"),
am.onlyif(with_stl and has_evt(Event.tree), "src/c4/yml/std/map.hpp"),
am.onlyif(with_stl and has_evt(Event.tree), "src/c4/yml/std/string.hpp"),
am.onlyif(with_stl and has_evt(Event.tree), "src/c4/yml/std/vector.hpp"),
am.onlyif(with_stl and has_evt(Event.tree), "src/c4/yml/std/std.hpp"),
"src/c4/yml/version.cpp",
"src/c4/yml/common.cpp",
"src/c4/yml/node_type.cpp",
"src/c4/yml/tag.cpp",
"src/c4/yml/tree.cpp",
"src/c4/yml/parse_engine.def.hpp",
"src/c4/yml/reference_resolver.cpp",
"src/c4/yml/parse.cpp",
"src/c4/yml/node.cpp",
am.onlyif(has_evt(Event.tree), "src/c4/yml/tree.cpp"),
am.onlyif(has_evt(Event.ints), "src_extra/c4/yml/extra/event_handler_ints.cpp"),
am.onlyif(has_evt(Event.ints_utils, Event.testsuite), "src_extra/c4/yml/extra/scalar.cpp"),
am.onlyif(has_evt(Event.ints_utils), "src_extra/c4/yml/extra/ints_utils.cpp"),
am.onlyif(has_evt(Event.ints_to_testsuite), "src_extra/c4/yml/extra/ints_to_testsuite.cpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/reference_resolver.cpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/parse.cpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/node.cpp"),
"src/c4/yml/preprocess.cpp",
"src/c4/yml/detail/checks.hpp",
"src/c4/yml/detail/print.hpp",
"src/c4/yml/yml.hpp",
"src/ryml.hpp",
am.onlyif(has_evt(Event.tree), "src/c4/yml/detail/checks.hpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/detail/print.hpp"),
am.onlyif(has_evt(Event.tree), "src/c4/yml/yml.hpp"),
am.onlyif(has_evt(Event.tree), "src/ryml.hpp"),
]
result = am.catfiles(srcfiles,
projdir,
@@ -143,14 +189,25 @@ INSTRUCTIONS:
def mkparser():
return am.mkparser(c4core=(True, "amalgamate c4core together with ryml"),
fastfloat=(True, "enable fastfloat library"),
stl=(True, "enable stl interop"))
p = am.mkparser(
c4core=(True, "amalgamate c4core together with ryml"),
fastfloat=(True, "enable fastfloat library"),
stl=(True, "enable stl interop")
)
default = [str(Event.tree)]
evtdoc = '. '.join([f"'{e}': {event_doc[e]}" for e in Event])
defaultdoc = ','.join([str(e) for e in default])
p.add_argument('-e', '--events', type=str, default=default, choices=[str(e) for e in Event], nargs="+",
help=f"""Specify which event handlers to include. Possible
values are: {evtdoc}. The default is {defaultdoc}.""")
return p
if __name__ == "__main__":
args = mkparser().parse_args()
args.events = [Event(e) for e in args.events] # is there a better way to do this?
amalgamate_ryml(filename=args.output,
with_c4core=args.c4core,
with_fastfloat=args.fastfloat,
with_stl=args.stl)
with_stl=args.stl,
events=args.events)

473
tools/yaml_events.cpp
View File

@@ -6,10 +6,17 @@
#include <c4/yml/event_handler_tree.hpp>
#include <c4/yml/parse_engine.def.hpp>
#endif
#include <test_suite/test_suite_events.hpp>
#include <test_suite/test_suite_event_handler.hpp>
#include <c4/yml/extra/string.hpp>
#include <c4/yml/extra/scalar.hpp>
#include <c4/yml/extra/event_handler_ints.hpp>
#include <c4/yml/extra/event_handler_testsuite.hpp>
#include <c4/yml/extra/ints_utils.hpp>
#include <c4/yml/extra/ints_to_testsuite.hpp>
#include <testsuite/testsuite_events.hpp>
#include <c4/fs/fs.hpp>
#include <cstdio>
#include <chrono>
#ifdef C4_EXCEPTIONS
#include <stdexcept>
@@ -19,80 +26,225 @@ std::jmp_buf jmp_env = {};
c4::csubstr jmp_msg = {};
#endif
C4_SUPPRESS_WARNING_GCC("-Wold-style-cast")
//-----------------------------------------------------------------------------
const char usage[] = R"(usage:
ryml-yaml-events [-s|-t] [-] # read from stdin (default)
ryml-yaml-events [-s|-t] <file> # read from file
The option can be one of the following:
-s emit events from source: parse the YAML source,
and directly emit events during the parse (ie, no
ryml tree is created). This is the default behavior
when the option is omitted.
-t events from tree: parse the YAML source, creating a
ryml tree. Once the tree is completely created, emit
the events from the created tree.
When the option is omitted, -s is assumed.
EXAMPLES:
$ ryml-yaml-events # emit events direct from stdin
$ ryml-yaml-events - # emit events direct from stdin
$ ryml-yaml-events -s - # emit events direct from stdin
$ ryml-yaml-events -t # parse stdin to tree, emit events from created tree
$ ryml-yaml-events -t - # parse stdin to tree, emit events from created tree
$ ryml-yaml-events <file> # emit events direct from file
$ ryml-yaml-events - <file> # emit events direct from file
$ ryml-yaml-events -s <file> # emit events direct from file
$ ryml-yaml-events -t <file> # parse file to tree, emit events from created tree
)";
using namespace c4;
using namespace c4::yml;
enum class evts_type
{
testsuite_src,
testsuite_ints,
testsuite_tree,
ryml_ints,
};
struct Args
{
csubstr filename = "-";
bool events_from_tree = false;
static bool parse(Args *args, int argc, const char *argv[]);
evts_type evts = evts_type::testsuite_src;
int ints_size = -1; // estimate by default
bool ints_size_force = false; // do not force the estimated size
static bool parse(Args *args, int argc, const char *argv[], int *errcode);
};
const char usage[] = R"(usage:
ryml-yaml-events <command> <options> [-] # read from stdin (default)
ryml-yaml-events <command> <options> <file> # read from file
The command must be one of the following:
testsuite_src,ts_src,tss
emit test suite events directly from source: parse the YAML
source, and directly emit events during the parse (ie, no
ryml tree is created). This is the default behavior when the
option is omitted.
testsuite_tree,ts_tree,tst
emit test suite events from tree: parse the YAML source,
creating a ryml tree. Once the tree is completely created,
emit the test suite events by iterating over the nodes of the
created tree.
testsuite_ints,ts_ints,tsi
emit test suite events from the ryml int events handler:
parse the YAML source to a container of ryml int events. Once
this is completed, emit the corresponding test suite events.
ryml_ints,ri
emit ryml int events: parse the YAML source to a container of
ryml int events. Once this is completed, print those same int
events.
The following options influence the behavior of the program:
--timings,--timing,-t
print task timings and size information (to stderr)
The following options influence the behavior of testsuite_ints and ryml_ints:
--ints-size <int-number>,-is <int-number>
when using int events, set the int event buffer size from this
value. use a negative value to force a conservative estimation
from a first run over the YAML source, and then multiply the
estimation by the symmetrical of that value. For example, -2
will result in 2*estimation. Default is -1.
--ints-size-force,-isf
when using int events, force the initial int event buffer size
to prevail: if this size is not large enough to accomodate the
actual number of events required from the YAML source, exit
with a nonzero error code. This is in contrast to the default
behavior, which consists of expanding the buffer as needed,
which requires two parses and two string copies of the
original source buffer.
EXAMPLES:
$ ryml-yaml-events ts_src # parse stdin to test suite events, then print the events
$ ryml-yaml-events ts_src - # parse stdin to test suite events, then print the events
$ ryml-yaml-events ts_src <file> # parse file to test suite events, then print the events
$ ryml-yaml-events ts_tree # parse stdin to ryml tree, emit test suite events from created tree
$ ryml-yaml-events ts_tree - # parse stdin to ryml tree, emit test suite events from created tree
$ ryml-yaml-events ts_tree <file> # parse file to ryml tree, emit test suite events from created tree
$ ryml-yaml-events ts_ints # parse stdin to ryml int events, emit test suite events from ryml int events
$ ryml-yaml-events ts_ints - # parse stdin to ryml int events, emit test suite events from ryml int events
$ ryml-yaml-events ts_ints <file> # parse file to ryml int events, emit test suite events from ryml int events
$ ryml-yaml-events ryml_ints # parse stdin to ryml int events, emit ryml int events
$ ryml-yaml-events ryml_ints - # parse stdin to ryml int events, emit ryml int events
$ ryml-yaml-events ryml_ints <file> # parse file to ryml int events, emit ryml int events
)";
//-----------------------------------------------------------------------------
using IntEvents = std::vector<extra::ievt::DataType>;
std::string load_file(csubstr filename);
std::string emit_events_from_tree(csubstr filename, substr filecontents);
std::string emit_events_direct(csubstr filename, substr filecontents);
extra::string emit_testsuite_events(csubstr filename, substr filecontents);
std::string emit_testsuite_events_from_tree(csubstr filename, substr filecontents);
std::string emit_testsuite_events_from_ints(csubstr filename, substr filecontents, IntEvents &evts, bool fail_size);
void emit_ints_events(csubstr filename, substr filecontents, IntEvents &evts, bool fail_size);
int estimate_ints_size(csubstr filecontents, int size);
Callbacks create_custom_callbacks();
bool timing_enabled = false;
double src_size = 0;
namespace stdc = std::chrono;
struct stopwatch
{
using clock_type = stdc::steady_clock;
const char *name;
clock_type::time_point start;
stopwatch(const char *name_)
{
if(!timing_enabled) return;
name = name_;
stack.emplace_back(this);
start = clock_type::now();
}
~stopwatch()
{
if(!timing_enabled) return;
stdc::duration<double, std::milli> delta = clock_type::now() - start;
for(stopwatch const* sw : stack)
fprintf(stderr, "%s:", sw->name);
fprintf(stderr, " %.6fms (%.3fMB/s)\n", delta.count(), src_size / delta.count() * 1.e-3);
stack.resize(stack.size()-1);
}
static std::vector<stopwatch*> stack;
};
std::vector<stopwatch*> stopwatch::stack;
#define STOPWATCH(name) stopwatch C4_XCAT(timer, __LINE__){name}
//-----------------------------------------------------------------------------
int main(int argc, const char *argv[])
{
Args args = {};
if(!Args::parse(&args, argc, argv))
return 1;
{
int errcode = 0;
if(!Args::parse(&args, argc, argv, &errcode))
return errcode;
}
set_callbacks(create_custom_callbacks());
C4_IF_EXCEPTIONS_(try, if(setjmp(jmp_env) == 0))
{
std::string src = load_file(args.filename);
const std::string events = args.events_from_tree ?
emit_events_from_tree(args.filename, to_substr(src))
:
emit_events_direct(args.filename, to_substr(src));
std::fwrite(events.data(), 1, events.size(), stdout);
std::string src;
{
STOPWATCH("load_file");
src = load_file(args.filename);
src_size = (double)src.size();
if(timing_enabled) fprintf(stderr, "src_size=%zuB\n", src.size());
}
STOPWATCH("process");
switch(args.evts)
{
case evts_type::testsuite_src:
{
extra::string evts;
{
STOPWATCH("testsuite_src");
evts = emit_testsuite_events(args.filename, to_substr(src));
}
{
STOPWATCH("print");
std::fwrite(evts.data(), 1, evts.size(), stdout);
}
break;
}
case evts_type::testsuite_tree:
{
std::string evts;
{
STOPWATCH("testsuite_tree");
evts = emit_testsuite_events_from_tree(args.filename, to_substr(src));
}
{
STOPWATCH("print");
std::fwrite(evts.data(), 1, evts.size(), stdout);
}
break;
}
case evts_type::testsuite_ints:
{
substr ssrc = to_substr(src);
int estimated_size = estimate_ints_size(ssrc, args.ints_size);
IntEvents int_evts((size_t)estimated_size);
std::string ts_evts;
{
STOPWATCH("testsuite_ints");
ts_evts = emit_testsuite_events_from_ints(args.filename, to_substr(src), int_evts, args.ints_size_force);
}
{
STOPWATCH("print");
std::fwrite(ts_evts.data(), 1, ts_evts.size(), stdout);
}
break;
}
case evts_type::ryml_ints:
{
substr ssrc = to_substr(src);
int estimated_size = estimate_ints_size(ssrc, args.ints_size);
IntEvents int_evts((size_t)estimated_size);
{
STOPWATCH("ryml_ints");
emit_ints_events(args.filename, to_substr(src), int_evts, args.ints_size_force);
}
break;
}
}
}
C4_IF_EXCEPTIONS_(catch(std::exception const&), else)
{
@@ -104,49 +256,206 @@ int main(int argc, const char *argv[])
//-----------------------------------------------------------------------------
std::string emit_events_from_tree(csubstr filename, substr filecontents)
std::string emit_testsuite_events_from_tree(csubstr filename, substr filecontents)
{
Tree tree(create_custom_callbacks());
tree.reserve(estimate_tree_capacity(filecontents));
parse_in_place(filename, filecontents, &tree);
return emit_events_from_tree<std::string>(tree);
{
STOPWATCH("tree_reserve");
tree.reserve(estimate_tree_capacity(filecontents));
}
{
STOPWATCH("parse");
parse_in_place(filename, filecontents, &tree);
}
{
STOPWATCH("emit_events");
std::string result = emit_events_from_tree<std::string>(tree);
return result;
}
}
std::string emit_events_direct(csubstr filename, substr filecontents)
extra::string emit_testsuite_events(csubstr filename, substr filecontents)
{
EventHandlerYamlStd::EventSink sink = {};
EventHandlerYamlStd handler(&sink, create_custom_callbacks());
ParseEngine<EventHandlerYamlStd> parser(&handler);
parser.parse_in_place_ev(filename, filecontents);
csubstr result = sink;
return std::string(result.str, result.len);
extra::EventHandlerTestSuite::EventSink sink = {};
extra::EventHandlerTestSuite handler(&sink, create_custom_callbacks());
ParseEngine<extra::EventHandlerTestSuite> parser(&handler);
{
STOPWATCH("parse");
parser.parse_in_place_ev(filename, filecontents);
}
return sink;
}
csubstr parse_events_ints(csubstr filename, substr filecontents, std::string &parsed, std::string &arena, IntEvents &evts, bool fail_size)
{
using I = extra::ievt::DataType;
using Handler = extra::EventHandlerInts;
Handler handler(create_custom_callbacks());
ParseEngine<Handler> parser(&handler);
substr src = filecontents;
if(!fail_size)
{
STOPWATCH("copy_src");
parsed.assign(filecontents.str, filecontents.len);
src = to_substr(parsed);
}
arena.resize(src.len);
handler.reset(src, to_substr(arena), evts.data(), (I)evts.size());
{
STOPWATCH("parse");
parser.parse_in_place_ev(filename, src);
}
size_t sz = (size_t)handler.required_size_events();
if(timing_enabled) fprintf(stderr, "current_size=%zu vs needed_size=%zu. arena_size=%zu\n", evts.size(), sz, arena.size());
if (!handler.fits_buffers())
{
RYML_CHECK(!fail_size);
{
STOPWATCH("resize");
evts.resize(sz);
arena.resize(handler.required_size_arena());
}
{
STOPWATCH("redo_copy_src");
parsed.assign(filecontents.str, filecontents.len);
src = to_substr(parsed);
}
handler.reset(src, to_substr(arena), evts.data(), (I)evts.size());
{
STOPWATCH("redo_parse");
parser.parse_in_place_ev(filename, src);
}
RYML_CHECK((size_t)handler.m_evt_pos == sz);
}
evts.resize(sz);
return src;
}
std::string emit_testsuite_events_from_ints(csubstr filename, substr filecontents, IntEvents &evts, bool fail_size)
{
using I = extra::ievt::DataType;
std::string buf;
std::string arena;
csubstr parsed;
{
STOPWATCH("events");
parsed = parse_events_ints(filename, filecontents, buf, arena, evts, fail_size);
}
std::string result;
{
STOPWATCH("emit");
extra::events_ints_to_testsuite(parsed, to_substr(arena), evts.data(), (I)evts.size(), &result);
}
return result;
}
void emit_ints_events(csubstr filename, substr filecontents, IntEvents &evts, bool fail_size)
{
using I = extra::ievt::DataType;
std::string buf;
std::string arena;
csubstr parsed;
{
STOPWATCH("events");
parsed = parse_events_ints(filename, filecontents, buf, arena, evts, fail_size);
}
{
STOPWATCH("emit");
extra::events_ints_print(parsed, to_substr(arena), evts.data(), (I)evts.size());
}
}
int estimate_ints_size(csubstr filecontents, int size)
{
if(size < 0)
{
STOPWATCH("estimate_size");
int est = extra::estimate_events_ints_size(filecontents);
if(timing_enabled) fprintf(stderr, "estimated_size=%d*%d=%d\n", -size, est, -size * est);
size = -size * est;
}
return size;
}
//-----------------------------------------------------------------------------
bool Args::parse(Args *args, int argc, const char *argv[])
// return true if the program should continue (eg -h should exit)
bool Args::parse(Args *args, int argc, const char *argv[], int *errcode)
{
if(argc > 3)
{
*errcode = 0;
auto argerr = [&](const char *msg){
std::printf(usage);
std::printf("ERROR: %s\n", msg);
*errcode = 1;
return false;
}
};
// set defaults
*args = {};
if(argc == 3)
// parse the command argument
if(argc < 2)
return argerr("must provide a command"); // LCOV_EXCL_LINE
bool has_cmd = false;
{
args->events_from_tree = (to_csubstr(argv[1]) == "-t");
args->filename = to_csubstr(argv[2]);
csubstr s = to_csubstr(argv[1]);
if(s == "testsuite_src" || s == "ts_src" || s == "tss")
{
args->evts = evts_type::testsuite_src;
has_cmd = true;
}
else if(s == "testsuite_tree" || s == "ts_tree" || s == "tst")
{
args->evts = evts_type::testsuite_tree;
has_cmd = true;
}
else if(s == "testsuite_ints" || s == "ts_ints" || s == "tsi")
{
args->evts = evts_type::testsuite_ints;
has_cmd = true;
}
else if(s == "ryml_ints" || s == "ri")
{
args->evts = evts_type::ryml_ints;
has_cmd = true;
}
else if(s == "--help" || s == "-h")
{
std::printf(usage);
return false;
}
}
else if(argc == 2)
if(!has_cmd)
return argerr("unknown command");
// parse other args
int i = 2; // cmd is mandatory at i=1, so start after that at i=2
for(; i < argc; ++i)
{
csubstr a = to_csubstr(argv[1]);
if(a == "-t")
args->events_from_tree = true;
else if(a == "-s")
args->events_from_tree = false;
csubstr arg = to_csubstr(argv[i]);
if(arg == "--help" || arg == "-h")
{
std::printf(usage);
return false;
}
else if(arg == "--timings" || arg == "--timing" || arg == "-t")
{
timing_enabled = true;
}
else if(arg == "--ints-size-force" || arg == "-isf")
{
args->ints_size_force = true;
}
else if(arg == "--ints-size" || arg == "-is")
{
if(i + 1 >= argc)
return argerr("ints-size value not given"); // LCOV_EXCL_LINE
else if(!atoi(to_csubstr(argv[i+1]), &args->ints_size))
return argerr("ints-size value fails to parse"); // LCOV_EXCL_LINE
++i; // shift 1
}
else
args->filename = a;
{
args->filename = arg;
}
}
return true;
}
@@ -154,19 +463,16 @@ bool Args::parse(Args *args, int argc, const char *argv[])
std::string load_file(csubstr filename)
{
if(filename == "-") // read from stdin
{
{ // LCOV_EXCL_START
std::string buf;
buf.reserve(128);
for(int c = std::getchar(); c != EOF; c = std::getchar())
{
buf.push_back(static_cast<char>(c));
if(buf.size() == buf.capacity())
buf.reserve(2u * (buf.capacity() >= 128u ? buf.capacity() : 128u));
}
return buf;
return buf; // LCOV_EXCL_STOP
}
if(!fs::path_exists(filename.str))
else if(!fs::path_exists(filename.str))
{
std::fprintf(stderr, "cannot find file: %s (cwd=%s)\n", filename.str, fs::cwd<std::string>().c_str());
std::fprintf(stderr, "%s: file not found (cwd=%s)\n", filename.str, fs::cwd<std::string>().c_str());
error("file not found");
}
return fs::file_get_contents<std::string>(filename.str);
@@ -204,4 +510,3 @@ Callbacks create_custom_callbacks()
};
return callbacks;
}