[FFmpeg-devel] [PATCH] fate: Allow running multiple rounds of tests with differing settings

[Martin Storsjö]

This can be used to run tests multple times, with e.g. differing
QEMU settings, by adding something like this to the FATE configuration
file:

    target_exec="qemu-aarch64-static"
    fate_targets="fate-checkasm fate-cpu"

    fate_environments="sve128 sve256 sve512"
    sve128_env="QEMU_CPU=max,sve128=on"
    sve256_env="QEMU_CPU=max,sve256=on"
    sve512_env="QEMU_CPU=max,sve512=on"

It's also possible to customize the target_exec command further
by injecting a sufficiently quoted variable into it, which then can
be updated for each run, e.g. target_exec="\$(CUR_EXEC_CMD)".

For each of the environment names in fate_environments, the tests
that are run get the name suffixed on the fate tests in the
test log and fate report, e.g. "fate-checkasm-h264dsp_sve128".
---
For adding the environment to the fate run, there's three
ways it can be done:
- (${curenv}; make fate)
- ${curenv} make fate
- make fate ${curenv}

The first has most freedom in what one can do, but requires explicit
"export VAR=value", while the other two are simpler for just setting
"VAR1=val1 VAR2=val2". For now, I picked the third form.
---
 tests/Makefile |  4 ++--
 tests/fate.sh  | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 444c09b3de..744dbcdfb3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -305,8 +305,8 @@ $(FATE): export PROGSUF = $(PROGSSUF)
 $(FATE): export EXECSUF = $(EXESUF)
 $(FATE): export HOSTEXECSUF = $(HOSTEXESUF)
 $(FATE): $(FATE_UTILS:%=tests/%$(HOSTEXESUF)) | $(FATE_OUTDIRS)
-	@echo "TEST    $(@:fate-%=%)"
-	$(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
+	@echo "TEST    $(@:fate-%=%)$(FATE_SUFFIX)"
+	$(Q)$(SRC_PATH)/tests/fate-run.sh $@$(FATE_SUFFIX) "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
 
 fate-list:
 	@printf '%s\n' $(sort $(FATE))
diff --git a/tests/fate.sh b/tests/fate.sh
index 95408ea109..c04eb41cbe 100755
--- a/tests/fate.sh
+++ b/tests/fate.sh
@@ -101,7 +101,19 @@ compile_extra()(
 fate()(
     test "$build_only" = "yes" && return
     cd ${build} || return
-    ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+    if [ -n "${fate_environments}" ]; then
+        ret=0
+        for e in ${fate_environments}; do
+            eval "curenv=\${${e}_env}"
+            echo Testing environment ${e}: ${curenv}
+            ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets} FATE_SUFFIX=_${e} ${curenv}
+            cur_ret=$?
+            test $cur_ret != 0 && ret=$cur_ret
+        done
+        return $ret
+    else
+        ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+    fi
 )
 
 clean(){
-- 
2.34.1