1
0
Fork 0
mirror of https://github.com/git/git.git synced 2024-10-31 06:17:56 +01:00
git/t/test-lib.sh

1159 lines
27 KiB
Bash
Raw Normal View History

#!/bin/sh
#
# Copyright (c) 2005 Junio C Hamano
#
test-lib.sh: Add explicit license detail, with change from GPLv2 to GPLv2+. Dear Junio, this is a resend of relicensing patch for test suite library, which was initially sent by Carl Worth. Since the time you sent me acks for this patch collected by you, I collected 8 additional acks as is documented at https://git.wiki.kernel.org/index.php/Test-lib_reclicensing. There are still three contributors missing: Bert Wesarg, Stephan Beyer and Bryan Donlan. The contributions of first two are clearly not copyrightable. I'm not sure about the copyrightability of Bryan Donlan's contributions (git log -p --author='Bryan Donlan' t/test-lib.sh). Carl told me that in your ack collection process you missed only three acks. So I wonder whether you already did some analysis of which contributions are copyrightable. If so, are the missing acks in the list bellow? Thanks Michal 8<--------8<--------8<-------- This file has had no explicit license information noted in it, but has clearly been created and modified according to the terms of GPLv2 as with the rest of the git code base. The purpose of relicensing is to allow other GPLv3+ projects (in particular, the notmuch project: http://notmuchmail.org) to use this same test-suite structure and to contribute changes back as well. Signed-off-by: Carl Worth <cworth@cworth.org> Signed-off-by: Michal Sojka <sojkam1@fel.cvut.cz> Acked-by: Alex Riesen <raa.lkml@gmail.com> Acked-by: Brandon Casey <drafnel@gmail.com> Acked-by: Clemens Buchacher <drizzd@aon.at> Acked-by: David Reiss <dreiss@facebook.com> Acked-by: Emil Sit <sit@emilsit.net> Acked-by: Eric Wong <normalperson@yhbt.net> Acked-by: Fredrik Kuivinen <frekui@gmail.com> Acked-by: Gerrit Pape <pape@smarden.org> Acked-by: Christian Couder <chriscool@tuxfamily.org> Acked-by: Jakub Narebski <jnareb@gmail.com> Acked-by: Jeff King <peff@peff.net> Acked-by: Johan Herland <johan@herland.net> Acked-by: Johannes Schindelin <Johannes.Schindelin@gmx.de> Acked-by: Johannes Sixt <j6t@kdbg.org> Acked-by: Jonathan Nieder <jrnieder@gmail.com> Acked-by: Josh Triplett <josh@joshtriplett.org> Acked-by: Junio C Hamano <gitster@pobox.com> Acked-by: Lea Wiemann <lewiemann@gmail.com> Acked-by: Markus Heidelberg <markus.heidelberg@web.de> Acked-by: Martin Waitz <tali@admingilde.org> Acked-by: Matthew Ogilvie <mmogilvi_git@miniinfo.net> Acked-by: Matthias Lederhofer <matled@gmx.net> Acked-by: Michael J Gruber <git@drmicha.warpmail.net> Acked-by: Michele Ballabio <barra_cuda@katamail.com> Acked-by: Miklos Vajna <vmiklos@frugalware.org> Acked-by: Nicolas Pitre <nico@fluxnic.net> Acked-by: Pavel Roskin <proski@gnu.org> Acked-by: Petr Baudis <pasky@ucw.cz> Acked-by: Pierre Habouzit <madcoder@debian.org> Acked-by: Robin Rosenberg <robin.rosenberg@dewire.com> Acked-by: Shawn O. Pearce <spearce@spearce.org> Acked-by: Stephen Boyd <bebarino@gmail.com> Acked-by: Sverre Rabbelier <srabbelier@gmail.com> Acked-by: Thomas Rast <trast@student.ethz.ch> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-04-16 15:53:59 +02:00
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/licenses/ .
# if --tee was passed, write the output not only to the terminal, but
# additionally to the file test-results/$BASENAME.out, too.
case "$GIT_TEST_TEE_STARTED, $* " in
done,*)
# do not redirect again
;;
*' --tee '*|*' --va'*)
mkdir -p test-results
BASE=test-results/$(basename "$0" .sh)
(GIT_TEST_TEE_STARTED=done ${SHELL-sh} "$0" "$@" 2>&1;
echo $? > $BASE.exit) | tee $BASE.out
test "$(cat $BASE.exit)" = 0
exit
;;
esac
# Keep the original TERM for say_color
ORIGINAL_TERM=$TERM
# For repeatability, reset the environment to known value.
LANG=C
LC_ALL=C
PAGER=cat
TZ=UTC
TERM=dumb
export LANG LC_ALL PAGER TERM TZ
EDITOR=:
unset VISUAL
unset EMAIL
unset $(perl -e '
my @env = keys %ENV;
my $ok = join("|", qw(
TRACE
DEBUG
USE_LOOKUP
TEST
.*_TEST
PROVE
VALGRIND
));
my @vars = grep(/^GIT_/ && !/^GIT_($ok)/o, @env);
print join("\n", @vars);
')
GIT_AUTHOR_EMAIL=author@example.com
GIT_AUTHOR_NAME='A U Thor'
GIT_COMMITTER_EMAIL=committer@example.com
GIT_COMMITTER_NAME='C O Mitter'
GIT_MERGE_VERBOSITY=5
export GIT_MERGE_VERBOSITY
export GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME
export GIT_COMMITTER_EMAIL GIT_COMMITTER_NAME
export EDITOR
# Protect ourselves from common misconfiguration to export
# CDPATH into the environment
unset CDPATH
unset GREP_OPTIONS
case $(echo $GIT_TRACE |tr "[A-Z]" "[a-z]") in
1|2|true)
echo "* warning: Some tests will not work if GIT_TRACE" \
"is set as to trace on STDERR ! *"
echo "* warning: Please set GIT_TRACE to something" \
"other than 1, 2 or true ! *"
;;
esac
# Convenience
#
# A regexp to match 5 and 40 hexdigits
_x05='[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]'
_x40="$_x05$_x05$_x05$_x05$_x05$_x05$_x05$_x05"
# Zero SHA-1
_z40=0000000000000000000000000000000000000000
# Line feed
LF='
'
# Each test should start with something like this, after copyright notices:
#
# test_description='Description of this test...
# This test checks if command xyzzy does the right thing...
# '
# . ./test-lib.sh
[ "x$ORIGINAL_TERM" != "xdumb" ] && (
TERM=$ORIGINAL_TERM &&
export TERM &&
[ -t 1 ] &&
tput bold >/dev/null 2>&1 &&
tput setaf 1 >/dev/null 2>&1 &&
tput sgr0 >/dev/null 2>&1
) &&
color=t
while test "$#" -ne 0
do
case "$1" in
-d|--d|--de|--deb|--debu|--debug)
debug=t; shift ;;
-i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate)
immediate=t; shift ;;
-l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests)
GIT_TEST_LONG=t; export GIT_TEST_LONG; shift ;;
-h|--h|--he|--hel|--help)
help=t; shift ;;
-v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
verbose=t; shift ;;
-q|--q|--qu|--qui|--quie|--quiet)
# Ignore --quiet under a TAP::Harness. Saying how many tests
# passed without the ok/not ok details is always an error.
test -z "$HARNESS_ACTIVE" && quiet=t; shift ;;
--with-dashes)
with_dashes=t; shift ;;
--no-color)
color=; shift ;;
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
--va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind)
valgrind=t; verbose=t; shift ;;
--tee)
shift ;; # was handled already
--root=*)
root=$(expr "z$1" : 'z[^=]*=\(.*\)')
shift ;;
*)
echo "error: unknown test option '$1'" >&2; exit 1 ;;
esac
done
if test -n "$color"; then
say_color () {
(
TERM=$ORIGINAL_TERM
export TERM
case "$1" in
error) tput bold; tput setaf 1;; # bold red
skip) tput bold; tput setaf 2;; # bold green
pass) tput setaf 2;; # green
info) tput setaf 3;; # brown
*) test -n "$quiet" && return;;
esac
shift
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
printf "%s" "$*"
tput sgr0
echo
)
}
else
say_color() {
test -z "$1" && test -n "$quiet" && return
shift
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
echo "$*"
}
fi
error () {
say_color error "error: $*"
GIT_EXIT_OK=t
exit 1
}
say () {
say_color info "$*"
}
test "${test_description}" != "" ||
error "Test script did not set test_description."
if test "$help" = "t"
then
echo "$test_description"
exit 0
fi
exec 5>&1
if test "$verbose" = "t"
then
exec 4>&2 3>&1
else
exec 4>/dev/null 3>/dev/null
fi
test_failure=0
test_count=0
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
test_fixed=0
test_broken=0
test_success=0
test_external_has_tap=0
die () {
code=$?
if test -n "$GIT_EXIT_OK"
then
exit $code
else
echo >&5 "FATAL: Unexpected exit with code $code"
exit 1
fi
}
GIT_EXIT_OK=
trap 'die' EXIT
# The semantics of the editor variables are that of invoking
# sh -c "$EDITOR \"$@\"" files ...
#
# If our trash directory contains shell metacharacters, they will be
# interpreted if we just set $EDITOR directly, so do a little dance with
# environment variables to work around this.
#
# In particular, quoting isn't enough, as the path may contain the same quote
# that we're using.
test_set_editor () {
FAKE_EDITOR="$1"
export FAKE_EDITOR
EDITOR='"$FAKE_EDITOR"'
export EDITOR
}
test_decode_color () {
awk '
function name(n) {
if (n == 0) return "RESET";
if (n == 1) return "BOLD";
if (n == 30) return "BLACK";
if (n == 31) return "RED";
if (n == 32) return "GREEN";
if (n == 33) return "YELLOW";
if (n == 34) return "BLUE";
if (n == 35) return "MAGENTA";
if (n == 36) return "CYAN";
if (n == 37) return "WHITE";
if (n == 40) return "BLACK";
if (n == 41) return "BRED";
if (n == 42) return "BGREEN";
if (n == 43) return "BYELLOW";
if (n == 44) return "BBLUE";
if (n == 45) return "BMAGENTA";
if (n == 46) return "BCYAN";
if (n == 47) return "BWHITE";
}
{
while (match($0, /\033\[[0-9;]*m/) != 0) {
printf "%s<", substr($0, 1, RSTART-1);
codes = substr($0, RSTART+2, RLENGTH-3);
if (length(codes) == 0)
printf "%s", name(0)
else {
n = split(codes, ary, ";");
sep = "";
for (i = 1; i <= n; i++) {
printf "%s%s", sep, name(ary[i]);
sep = ";"
}
}
printf ">";
$0 = substr($0, RSTART + RLENGTH, length($0) - RSTART - RLENGTH + 1);
}
print
}
'
}
nul_to_q () {
perl -pe 'y/\000/Q/'
}
q_to_nul () {
perl -pe 'y/Q/\000/'
}
q_to_cr () {
tr Q '\015'
}
q_to_tab () {
tr Q '\011'
}
append_cr () {
sed -e 's/$/Q/' | tr Q '\015'
}
remove_cr () {
tr '\015' Q | sed -e 's/Q$//'
}
# In some bourne shell implementations, the "unset" builtin returns
# nonzero status when a variable to be unset was not set in the first
# place.
#
# Use sane_unset when that should not be considered an error.
sane_unset () {
unset "$@"
return 0
}
test_tick () {
if test -z "${test_tick+set}"
then
test_tick=1112911993
else
test_tick=$(($test_tick + 60))
fi
GIT_COMMITTER_DATE="$test_tick -0700"
GIT_AUTHOR_DATE="$test_tick -0700"
export GIT_COMMITTER_DATE GIT_AUTHOR_DATE
}
# Call test_commit with the arguments "<message> [<file> [<contents>]]"
#
# This will commit a file with the given contents and the given commit
# message. It will also add a tag with <message> as name.
#
# Both <file> and <contents> default to <message>.
test_commit () {
file=${2:-"$1.t"}
echo "${3-$1}" > "$file" &&
git add "$file" &&
test_tick &&
git commit -m "$1" &&
git tag "$1"
}
# Call test_merge with the arguments "<message> <commit>", where <commit>
# can be a tag pointing to the commit-to-merge.
test_merge () {
test_tick &&
git merge -m "$1" "$2" &&
git tag "$1"
}
# This function helps systems where core.filemode=false is set.
# Use it instead of plain 'chmod +x' to set or unset the executable bit
# of a file in the working directory and add it to the index.
test_chmod () {
chmod "$@" &&
git update-index --add "--chmod=$@"
}
# Unset a configuration variable, but don't fail if it doesn't exist.
test_unconfig () {
git config --unset-all "$@"
config_status=$?
case "$config_status" in
5) # ok, nothing to unset
config_status=0
;;
esac
return $config_status
}
# Set git config, automatically unsetting it after the test is over.
test_config () {
test_when_finished "test_unconfig '$1'" &&
git config "$@"
}
# Use test_set_prereq to tell that a particular prerequisite is available.
# The prerequisite can later be checked for in two ways:
#
# - Explicitly using test_have_prereq.
#
# - Implicitly by specifying the prerequisite tag in the calls to
# test_expect_{success,failure,code}.
#
# The single parameter is the prerequisite tag (a simple word, in all
# capital letters by convention).
test_set_prereq () {
satisfied="$satisfied$1 "
}
satisfied=" "
test_have_prereq () {
# prerequisites can be concatenated with ','
save_IFS=$IFS
IFS=,
set -- $*
IFS=$save_IFS
total_prereq=0
ok_prereq=0
missing_prereq=
for prerequisite
do
total_prereq=$(($total_prereq + 1))
case $satisfied in
*" $prerequisite "*)
ok_prereq=$(($ok_prereq + 1))
;;
*)
# Keep a list of missing prerequisites
if test -z "$missing_prereq"
then
missing_prereq=$prerequisite
else
missing_prereq="$prerequisite,$missing_prereq"
fi
esac
done
test $total_prereq = $ok_prereq
}
test_declared_prereq () {
case ",$test_prereq," in
*,$1,*)
return 0
;;
esac
return 1
}
# You are not expected to call test_ok_ and test_failure_ directly, use
# the text_expect_* functions instead.
test_ok_ () {
test_success=$(($test_success + 1))
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
say_color "" "ok $test_count - $@"
}
test_failure_ () {
test_failure=$(($test_failure + 1))
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
say_color error "not ok - $test_count $1"
shift
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
echo "$@" | sed -e 's/^/# /'
test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
}
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
test_known_broken_ok_ () {
test_fixed=$(($test_fixed+1))
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
say_color "" "ok $test_count - $@ # TODO known breakage"
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
}
test_known_broken_failure_ () {
test_broken=$(($test_broken+1))
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
say_color skip "not ok $test_count - $@ # TODO known breakage"
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
}
test_debug () {
test "$debug" = "" || eval "$1"
}
test_eval_ () {
# This is a separate function because some tests use
# "return" to end a test_expect_success block early.
eval >&3 2>&4 "$*"
}
test_run_ () {
test_cleanup=:
expecting_failure=$2
test_eval_ "$1"
eval_ret=$?
if test -z "$immediate" || test $eval_ret = 0 || test -n "$expecting_failure"
then
test_eval_ "$test_cleanup"
fi
if test "$verbose" = "t" && test -n "$HARNESS_ACTIVE"; then
echo ""
fi
return "$eval_ret"
}
test_skip () {
test_count=$(($test_count+1))
to_skip=
for skp in $GIT_SKIP_TESTS
do
case $this_test.$test_count in
$skp)
to_skip=t
break
esac
done
if test -z "$to_skip" && test -n "$test_prereq" &&
! test_have_prereq "$test_prereq"
then
to_skip=t
fi
case "$to_skip" in
t)
of_prereq=
if test "$missing_prereq" != "$test_prereq"
then
of_prereq=" of $test_prereq"
fi
say_color skip >&3 "skipping test: $@"
say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})"
: true
;;
*)
false
;;
esac
}
test_expect_failure () {
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
test "$#" = 2 ||
error "bug in the test script: not 2 or 3 parameters to test-expect-failure"
export test_prereq
if ! test_skip "$@"
then
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
say >&3 "checking known breakage: $2"
if test_run_ "$2" expecting_failure
then
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
test_known_broken_ok_ "$1"
else
test_known_broken_failure_ "$1"
fi
fi
echo >&3 ""
}
test_expect_success () {
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
test "$#" = 2 ||
error "bug in the test script: not 2 or 3 parameters to test-expect-success"
export test_prereq
if ! test_skip "$@"
then
say >&3 "expecting success: $2"
if test_run_ "$2"
then
test_ok_ "$1"
else
test_failure_ "$@"
fi
fi
echo >&3 ""
}
# test_external runs external test scripts that provide continuous
# test output about their progress, and succeeds/fails on
# zero/non-zero exit code. It outputs the test output on stdout even
# in non-verbose mode, and announces the external script with "# run
# <n>: ..." before running it. When providing relative paths, keep in
# mind that all scripts run in "trash directory".
# Usage: test_external description command arguments...
# Example: test_external 'Perl API' perl ../path/to/test.pl
test_external () {
test "$#" = 4 && { test_prereq=$1; shift; } || test_prereq=
test "$#" = 3 ||
error >&5 "bug in the test script: not 3 or 4 parameters to test_external"
descr="$1"
shift
export test_prereq
if ! test_skip "$descr" "$@"
then
# Announce the script to reduce confusion about the
# test output that follows.
say_color "" "# run $test_count: $descr ($*)"
# Export TEST_DIRECTORY, TRASH_DIRECTORY and GIT_TEST_LONG
# to be able to use them in script
export TEST_DIRECTORY TRASH_DIRECTORY GIT_TEST_LONG
# Run command; redirect its stderr to &4 as in
# test_run_, but keep its stdout on our stdout even in
# non-verbose mode.
"$@" 2>&4
if [ "$?" = 0 ]
then
if test $test_external_has_tap -eq 0; then
test_ok_ "$descr"
else
say_color "" "# test_external test $descr was ok"
test_success=$(($test_success + 1))
fi
else
if test $test_external_has_tap -eq 0; then
test_failure_ "$descr" "$@"
else
say_color error "# test_external test $descr failed: $@"
test_failure=$(($test_failure + 1))
fi
fi
fi
}
# Like test_external, but in addition tests that the command generated
# no output on stderr.
test_external_without_stderr () {
# The temporary file has no (and must have no) security
# implications.
tmp=${TMPDIR:-/tmp}
stderr="$tmp/git-external-stderr.$$.tmp"
test_external "$@" 4> "$stderr"
[ -f "$stderr" ] || error "Internal error: $stderr disappeared."
descr="no stderr: $1"
shift
say >&3 "# expecting no stderr from previous command"
if [ ! -s "$stderr" ]; then
rm "$stderr"
if test $test_external_has_tap -eq 0; then
test_ok_ "$descr"
else
say_color "" "# test_external_without_stderr test $descr was ok"
test_success=$(($test_success + 1))
fi
else
if [ "$verbose" = t ]; then
output=`echo; echo "# Stderr is:"; cat "$stderr"`
else
output=
fi
# rm first in case test_failure exits.
rm "$stderr"
if test $test_external_has_tap -eq 0; then
test_failure_ "$descr" "$@" "$output"
else
say_color error "# test_external_without_stderr test $descr failed: $@: $output"
test_failure=$(($test_failure + 1))
fi
fi
}
# debugging-friendly alternatives to "test [-f|-d|-e]"
# The commands test the existence or non-existence of $1. $2 can be
# given to provide a more precise diagnosis.
test_path_is_file () {
if ! [ -f "$1" ]
then
echo "File $1 doesn't exist. $*"
false
fi
}
test_path_is_dir () {
if ! [ -d "$1" ]
then
echo "Directory $1 doesn't exist. $*"
false
fi
}
test_path_is_missing () {
if [ -e "$1" ]
then
echo "Path exists:"
ls -ld "$1"
if [ $# -ge 1 ]; then
echo "$*"
fi
false
fi
}
# test_line_count checks that a file has the number of lines it
# ought to. For example:
#
# test_expect_success 'produce exactly one line of output' '
# do something >output &&
# test_line_count = 1 output
# '
#
# is like "test $(wc -l <output) = 1" except that it passes the
# output through when the number of lines is wrong.
test_line_count () {
if test $# != 3
then
error "bug in the test script: not 3 parameters to test_line_count"
elif ! test $(wc -l <"$3") "$1" "$2"
then
echo "test_line_count: line count for $3 !$1 $2"
cat "$3"
return 1
fi
}
# This is not among top-level (test_expect_success | test_expect_failure)
# but is a prefix that can be used in the test script, like:
#
# test_expect_success 'complain and die' '
# do something &&
# do something else &&
# test_must_fail git checkout ../outerspace
# '
#
# Writing this as "! git checkout ../outerspace" is wrong, because
# the failure could be due to a segv. We want a controlled failure.
test_must_fail () {
"$@"
exit_code=$?
if test $exit_code = 0; then
echo >&2 "test_must_fail: command succeeded: $*"
return 1
elif test $exit_code -gt 129 -a $exit_code -le 192; then
echo >&2 "test_must_fail: died by signal: $*"
return 1
elif test $exit_code = 127; then
echo >&2 "test_must_fail: command not found: $*"
return 1
fi
return 0
}
# Similar to test_must_fail, but tolerates success, too. This is
# meant to be used in contexts like:
#
# test_expect_success 'some command works without configuration' '
# test_might_fail git config --unset all.configuration &&
# do something
# '
#
# Writing "git config --unset all.configuration || :" would be wrong,
# because we want to notice if it fails due to segv.
test_might_fail () {
"$@"
exit_code=$?
if test $exit_code -gt 129 -a $exit_code -le 192; then
echo >&2 "test_might_fail: died by signal: $*"
return 1
elif test $exit_code = 127; then
echo >&2 "test_might_fail: command not found: $*"
return 1
fi
return 0
}
# Similar to test_must_fail and test_might_fail, but check that a
# given command exited with a given exit code. Meant to be used as:
#
# test_expect_success 'Merge with d/f conflicts' '
# test_expect_code 1 git merge "merge msg" B master
# '
test_expect_code () {
want_code=$1
shift
"$@"
exit_code=$?
if test $exit_code = $want_code
then
return 0
fi
echo >&2 "test_expect_code: command exited with $exit_code, we wanted $want_code $*"
return 1
}
# test_cmp is a helper function to compare actual and expected output.
# You can use it like:
#
# test_expect_success 'foo works' '
# echo expected >expected &&
# foo >actual &&
# test_cmp expected actual
# '
#
# This could be written as either "cmp" or "diff -u", but:
# - cmp's output is not nearly as easy to read as diff -u
# - not all diff versions understand "-u"
test_cmp() {
$GIT_TEST_CMP "$@"
}
# This function can be used to schedule some commands to be run
# unconditionally at the end of the test to restore sanity:
#
# test_expect_success 'test core.capslock' '
# git config core.capslock true &&
# test_when_finished "git config --unset core.capslock" &&
# hello world
# '
#
# That would be roughly equivalent to
#
# test_expect_success 'test core.capslock' '
# git config core.capslock true &&
# hello world
# git config --unset core.capslock
# '
#
# except that the greeting and config --unset must both succeed for
# the test to pass.
#
# Note that under --immediate mode, no clean-up is done to help diagnose
# what went wrong.
test_when_finished () {
test_cleanup="{ $*
} && (exit \"\$eval_ret\"); eval_ret=\$?; $test_cleanup"
}
# Most tests can use the created repository, but some may need to create more.
# Usage: test_create_repo <directory>
test_create_repo () {
test "$#" = 1 ||
error "bug in the test script: not 1 parameter to test-create-repo"
repo="$1"
mkdir -p "$repo"
(
cd "$repo" || error "Cannot setup test environment"
"$GIT_EXEC_PATH/git-init" "--template=$GIT_BUILD_DIR/templates/blt/" >&3 2>&4 ||
error "cannot run git init -- have you built things yet?"
mv .git/hooks .git/hooks-disabled
) || exit
}
test_done () {
GIT_EXIT_OK=t
if test -z "$HARNESS_ACTIVE"; then
test_results_dir="$TEST_DIRECTORY/test-results"
mkdir -p "$test_results_dir"
test_results_path="$test_results_dir/${0%.sh}-$$.counts"
cat >>"$test_results_path" <<-EOF
total $test_count
success $test_success
fixed $test_fixed
broken $test_broken
failed $test_failure
EOF
fi
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
if test "$test_fixed" != 0
then
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
say_color pass "# fixed $test_fixed known breakage(s)"
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
fi
if test "$test_broken" != 0
then
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
say_color error "# still have $test_broken known breakage(s)"
msg="remaining $(($test_count-$test_broken)) test(s)"
else
msg="$test_count test(s)"
Sane use of test_expect_failure Originally, test_expect_failure was designed to be the opposite of test_expect_success, but this was a bad decision. Most tests run a series of commands that leads to the single command that needs to be tested, like this: test_expect_{success,failure} 'test title' ' setup1 && setup2 && setup3 && what is to be tested ' And expecting a failure exit from the whole sequence misses the point of writing tests. Your setup$N that are supposed to succeed may have failed without even reaching what you are trying to test. The only valid use of test_expect_failure is to check a trivial single command that is expected to fail, which is a minority in tests of Porcelain-ish commands. This large-ish patch rewrites all uses of test_expect_failure to use test_expect_success and rewrites the condition of what is tested, like this: test_expect_success 'test title' ' setup1 && setup2 && setup3 && ! this command should fail ' test_expect_failure is redefined to serve as a reminder that that test *should* succeed but due to a known breakage in git it currently does not pass. So if git-foo command should create a file 'bar' but you discovered a bug that it doesn't, you can write a test like this: test_expect_failure 'git-foo should create bar' ' rm -f bar && git foo && test -f bar ' This construct acts similar to test_expect_success, but instead of reporting "ok/FAIL" like test_expect_success does, the outcome is reported as "FIXED/still broken". Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-02-01 10:50:53 +01:00
fi
case "$test_failure" in
0)
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
# Maybe print SKIP message
[ -z "$skip_all" ] || skip_all=" # SKIP $skip_all"
if test $test_external_has_tap -eq 0; then
say_color pass "# passed all $msg"
say "1..$test_count$skip_all"
fi
test -d "$remove_trash" &&
cd "$(dirname "$remove_trash")" &&
rm -rf "$(basename "$remove_trash")"
exit 0 ;;
*)
if test $test_external_has_tap -eq 0; then
say_color error "# failed $test_failure among $msg"
say "1..$test_count"
fi
test-lib: Adjust output to be valid TAP format TAP, the Test Anything Protocol, is a simple text-based interface between testing modules in a test harness. test-lib.sh's output was already very close to being valid TAP. This change brings it all the way there. Before: $ ./t0005-signals.sh * ok 1: sigchain works * passed all 1 test(s) And after: $ ./t0005-signals.sh ok 1 - sigchain works # passed all 1 test(s) 1..1 The advantage of using TAP is that any program that reads the format (a "test harness") can run the tests. The most popular of these is the prove(1) utility that comes with Perl. It can run tests in parallel, display colored output, format the output to console, file, HTML etc., and much more. An example: $ prove ./t0005-signals.sh ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU) Result: PASS prove(1) gives you human readable output without being too verbose. Running the test suite in parallel with `make test -j15` produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh` makes it easy to follow what's going on. All this patch does is re-arrange the output a bit so that it conforms with the TAP spec, everything that the test suite did before continues to work. That includes aggregating results in t/test-results/, the --verbose, --debug and other options for tests, and the test color output. TAP harnesses ignore everything that they don't know about, so running the tests with --verbose works: $ prove ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Terminated ./t0005-signals.sh .. ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU) Result: PASS Just supply the -v option to prove itself to get all the verbose output that it suppresses: $ prove -v ./t0005-signals.sh :: --verbose --debug ./t0005-signals.sh .. Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/ expecting success: test-sigchain >actual case "$?" in 143) true ;; # POSIX w/ SIGTERM=15 3) true ;; # Windows *) false ;; esac && test_cmp expect actual Terminated ok 1 - sigchain works # passed all 1 test(s) 1..1 ok All tests successful. Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU) Result: PASS As a further example, consider this test script that uses a lot of test-lib.sh features by Jakub Narebski: #!/bin/sh test_description='this is a sample test. This test is here to see various test outputs.' . ./test-lib.sh say 'diagnostic message' test_expect_success 'true test' 'true' test_expect_success 'false test' 'false' test_expect_failure 'true test (todo)' 'true' test_expect_failure 'false test (todo)' 'false' test_debug 'echo "debug message"' test_done The output of that was previously: * diagnostic message # yellow * ok 1: true test * FAIL 2: false test # bold red false * FIXED 3: true test (todo) * still broken 4: false test (todo) # bold green * fixed 1 known breakage(s) # green * still have 1 known breakage(s) # bold red * failed 1 among remaining 3 test(s) # bold red But is now: diagnostic message # yellow ok 1 - true test not ok - 2 false test # bold red # false ok 3 - true test (todo) # TODO known breakage not ok 4 - false test (todo) # TODO known breakage # bold green # fixed 1 known breakage(s) # green # still have 1 known breakage(s) # bold red # failed 1 among remaining 3 test(s) # bold red 1..4 All the coloring is preserved when the test is run manually. Under prove(1) the test performs as expected, even with --debug and --verbose options: $ prove ./example.sh :: --debug --verbose ./example.sh .. Dubious, test returned 1 (wstat 256, 0x100) Failed 1/4 subtests (1 TODO test unexpectedly succeeded) Test Summary Report ------------------- ./example.sh (Wstat: 256 Tests: 4 Failed: 1) Failed test: 2 TODO passed: 3 Non-zero exit status: 1 Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU) Result: FAIL The TAP harness itself doesn't get confused by the color output, they aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1). Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 23:52:12 +02:00
exit 1 ;;
esac
}
# Test the binaries we have just built. The tests are kept in
# t/ subdirectory and are run in 'trash directory' subdirectory.
if test -z "$TEST_DIRECTORY"
then
# We allow tests to override this, in case they want to run tests
# outside of t/, e.g. for running tests on the test library
# itself.
TEST_DIRECTORY=$(pwd)
fi
GIT_BUILD_DIR="$TEST_DIRECTORY"/..
if test -n "$valgrind"
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
then
make_symlink () {
test -h "$2" &&
test "$1" = "$(readlink "$2")" || {
# be super paranoid
if mkdir "$2".lock
then
rm -f "$2" &&
ln -s "$1" "$2" &&
rm -r "$2".lock
else
while test -d "$2".lock
do
say "Waiting for lock on $2."
sleep 1
done
fi
}
}
make_valgrind_symlink () {
# handle only executables, unless they are shell libraries that
# need to be in the exec-path. We will just use "#!" as a
# guess for a shell-script, since we have no idea what the user
# may have configured as the shell path.
test -x "$1" ||
test "#!" = "$(head -c 2 <"$1")" ||
return;
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
base=$(basename "$1")
symlink_target=$GIT_BUILD_DIR/$base
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
# do not override scripts
if test -x "$symlink_target" &&
test ! -d "$symlink_target" &&
test "#!" != "$(head -c 2 < "$symlink_target")"
then
symlink_target=../valgrind.sh
fi
case "$base" in
*.sh|*.perl)
symlink_target=../unprocessed-script
esac
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
# create the link, or replace it if it is out of date
make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
}
# override all git executables in TEST_DIRECTORY/..
GIT_VALGRIND=$TEST_DIRECTORY/valgrind
mkdir -p "$GIT_VALGRIND"/bin
for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/test-*
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
do
make_valgrind_symlink $file
done
# special-case the mergetools loadables
make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools"
OLDIFS=$IFS
IFS=:
for path in $PATH
do
ls "$path"/git-* 2> /dev/null |
while read file
do
make_valgrind_symlink "$file"
done
done
IFS=$OLDIFS
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
PATH=$GIT_VALGRIND/bin:$PATH
GIT_EXEC_PATH=$GIT_VALGRIND/bin
export GIT_VALGRIND
elif test -n "$GIT_TEST_INSTALLED" ; then
GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path) ||
error "Cannot run git from $GIT_TEST_INSTALLED."
PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR:$PATH
GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
else # normal case, use ../bin-wrappers only unless $with_dashes:
git_bin_dir="$GIT_BUILD_DIR/bin-wrappers"
if ! test -x "$git_bin_dir/git" ; then
if test -z "$with_dashes" ; then
say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH"
fi
with_dashes=t
fi
PATH="$git_bin_dir:$PATH"
GIT_EXEC_PATH=$GIT_BUILD_DIR
if test -n "$with_dashes" ; then
PATH="$GIT_BUILD_DIR:$PATH"
fi
Add valgrind support in test scripts This patch adds the ability to use valgrind's memcheck tool to diagnose memory problems in Git while running the test scripts. It requires valgrind 3.4.0 or newer. It works by creating symlinks to a valgrind script, which have the same name as our Git binaries, and then putting that directory in front of the test script's PATH as well as set GIT_EXEC_PATH to that directory. Git scripts are symlinked from that directory directly. That way, Git binaries called by Git scripts are valgrinded, too. Valgrind can be used by specifying "GIT_TEST_OPTS=--valgrind" in the make invocation. Any invocation of git that finds any errors under valgrind will exit with failure code 126. Any valgrind output will go to the usual stderr channel for tests (i.e., /dev/null, unless -v has been specified). If you need to pass options to valgrind -- you might want to run another tool than memcheck, for example -- you can set the environment variable GIT_VALGRIND_OPTIONS. A few default suppressions are included, since libz seems to trigger quite a few false positives. We'll assume that libz works and that we can ignore any errors which are reported there. Note: it is safe to run the valgrind tests in parallel, as the links in t/valgrind/bin/ are created using proper locking. Initial patch and all the hard work by Jeff King. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-02-04 00:25:59 +01:00
fi
GIT_TEMPLATE_DIR="$GIT_BUILD_DIR"/templates/blt
unset GIT_CONFIG
GIT_CONFIG_NOSYSTEM=1
GIT_ATTR_NOSYSTEM=1
export PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR GIT_CONFIG_NOSYSTEM GIT_ATTR_NOSYSTEM
. "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS
if test -z "$GIT_TEST_CMP"
then
if test -n "$GIT_TEST_CMP_USE_COPIED_CONTEXT"
then
GIT_TEST_CMP="$DIFF -c"
else
GIT_TEST_CMP="$DIFF -u"
fi
fi
GITPERLLIB="$GIT_BUILD_DIR"/perl/blib/lib:"$GIT_BUILD_DIR"/perl/blib/arch/auto/Git
export GITPERLLIB
test -d "$GIT_BUILD_DIR"/templates/blt || {
error "You haven't built things yet, have you?"
}
if test -z "$GIT_TEST_INSTALLED" && test -z "$NO_PYTHON"
then
GITPYTHONLIB="$GIT_BUILD_DIR/git_remote_helpers/build/lib"
export GITPYTHONLIB
test -d "$GIT_BUILD_DIR"/git_remote_helpers/build || {
error "You haven't built git_remote_helpers yet, have you?"
}
fi
if ! test -x "$GIT_BUILD_DIR"/test-chmtime; then
echo >&2 'You need to build test-chmtime:'
echo >&2 'Run "make test-chmtime" in the source (toplevel) directory'
exit 1
fi
# Test repository
test="trash directory.$(basename "$0" .sh)"
test -n "$root" && test="$root/$test"
case "$test" in
/*) TRASH_DIRECTORY="$test" ;;
*) TRASH_DIRECTORY="$TEST_DIRECTORY/$test" ;;
esac
test ! -z "$debug" || remove_trash=$TRASH_DIRECTORY
rm -fr "$test" || {
GIT_EXIT_OK=t
echo >&5 "FATAL: Cannot prepare test area"
exit 1
}
HOME="$TRASH_DIRECTORY"
export HOME
test_create_repo "$test"
# Use -P to resolve symlinks in our working directory so that the cwd
# in subprocesses like git equals our $PWD (for pathname comparisons).
cd -P "$test" || exit 1
this_test=${0##*/}
this_test=${this_test%%-*}
for skp in $GIT_SKIP_TESTS
do
case "$this_test" in
$skp)
say_color skip >&3 "skipping test $this_test altogether"
skip_all="skip all tests in $this_test"
test_done
esac
done
# Provide an implementation of the 'yes' utility
yes () {
if test $# = 0
then
y=y
else
y="$*"
fi
while echo "$y"
do
:
done
}
# Fix some commands on Windows
case $(uname -s) in
*MINGW*)
# Windows has its own (incompatible) sort and find
sort () {
/usr/bin/sort "$@"
}
find () {
/usr/bin/find "$@"
}
sum () {
md5sum "$@"
}
# git sees Windows-style pwd
pwd () {
builtin pwd -W
}
# no POSIX permissions
# backslashes in pathspec are converted to '/'
# exec does not inherit the PID
test_set_prereq MINGW
test_set_prereq SED_STRIPS_CR
;;
*CYGWIN*)
test_set_prereq POSIXPERM
test_set_prereq EXECKEEPSPID
test_set_prereq NOT_MINGW
test_set_prereq SED_STRIPS_CR
;;
*)
test_set_prereq POSIXPERM
test_set_prereq BSLASHPSPEC
test_set_prereq EXECKEEPSPID
test_set_prereq NOT_MINGW
;;
esac
test -z "$NO_PERL" && test_set_prereq PERL
test -z "$NO_PYTHON" && test_set_prereq PYTHON
test -n "$USE_LIBPCRE" && test_set_prereq LIBPCRE
# Can we rely on git's output in the C locale?
if test -n "$GETTEXT_POISON"
then
GIT_GETTEXT_POISON=YesPlease
export GIT_GETTEXT_POISON
else
test_set_prereq C_LOCALE_OUTPUT
fi
# Use this instead of test_cmp to compare files that contain expected and
# actual output from git commands that can be translated. When running
# under GETTEXT_POISON this pretends that the command produced expected
# results.
test_i18ncmp () {
test -n "$GETTEXT_POISON" || test_cmp "$@"
}
# Use this instead of "grep expected-string actual" to see if the
# output from a git command that can be translated either contains an
# expected string, or does not contain an unwanted one. When running
# under GETTEXT_POISON this pretends that the command produced expected
# results.
test_i18ngrep () {
if test -n "$GETTEXT_POISON"
then
: # pretend success
elif test "x!" = "x$1"
then
shift
! grep "$@"
else
grep "$@"
fi
}
# test whether the filesystem supports symbolic links
ln -s x y 2>/dev/null && test -h y 2>/dev/null && test_set_prereq SYMLINKS
rm -f y
# When the tests are run as root, permission tests will report that
# things are writable when they shouldn't be.
test -w / || test_set_prereq SANITY