git/t/t7300-clean.sh at 18d8c26930acbba1627d2a8b7323be30ac2bd9bb - mirrors/git - Incest Forge: Beyond sex. We incest.

mirrors/git

mirror of https://github.com/git/git.git synced 2024-11-01 06:47:52 +01:00

Erik Elfström 0179ca7a62 clean: improve performance when removing lots of directories

"git clean" uses resolve_gitlink_ref() to check for the presence of
nested git repositories, but it has the drawback of creating a
ref_cache entry for every directory that should potentially be
cleaned. The linear search through the ref_cache list causes a massive
performance hit for large number of directories.

Modify clean.c:remove_dirs to use setup.c:is_git_directory and
setup.c:read_gitfile_gently instead.

Both these functions will open files and parse contents when they find
something that looks like a git repository. This is ok from a
performance standpoint since finding repository candidates should be
comparatively rare.

Using is_git_directory and read_gitfile_gently should give a more
standardized check for what is and what isn't a git repository but
also gives three behavioral changes.

The first change is that we will now detect and avoid cleaning empty
nested git repositories (only init run). This is desirable.

Second, we will no longer die when cleaning a file named ".git" with
garbage content (it will be cleaned instead). This is also desirable.

The last change is that we will detect and avoid cleaning empty bare
repositories that have been placed in a directory named ".git". This
is not desirable but should have no real user impact since we already
fail to clean non-empty bare repositories in the same scenario. This
is thus deemed acceptable.

On top of this we add some extra precautions. If read_gitfile_gently
fails to open the git file, read the git file or verify the path in
the git file we assume that the path with the git file is a valid
repository and avoid cleaning.

Update t7300 to reflect these changes in behavior.

The time to clean an untracked directory containing 100000 sub
directories went from 61s to 1.7s after this change.

Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Erik Elfström <erik.elfstrom@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

2015-06-15 13:14:24 -07:00

664 lines

14 KiB

Bash

Executable file

Raw Blame History

 #!/bin/sh
 #
 # Copyright (c) 2007 Michael Spang
 #
 test_description='git clean basic tests'
 . ./test-lib.sh
 git config clean.requireForce no
 test_expect_success 'setup' '
 	mkdir -p src &&
 	touch src/part1.c Makefile &&
 	echo build >.gitignore &&
 	echo \*.o >>.gitignore &&
 	git add . &&
 	git commit -m setup &&
 	touch src/part2.c README &&
 	git add .
 '
 test_expect_success 'git clean with skip-worktree .gitignore' '
 	git update-index --skip-worktree .gitignore &&
 	rm .gitignore &&
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test ! -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so &&
 	git update-index --no-skip-worktree .gitignore &&
 	git checkout .gitignore
 '
 test_expect_success 'git clean' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test ! -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean src/' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean src/ &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test ! -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean src/ src/' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean src/ src/ &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test ! -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean with prefix' '
 	mkdir -p build docs src/test &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so src/test/1.c &&
 	(cd src/ && git clean) &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test ! -f src/part3.c &&
 	test -f src/test/1.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success C_LOCALE_OUTPUT 'git clean with relative prefix' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	would_clean=$(
 		cd docs &&
 		git clean -n ../src |
 		sed -n -e "s|^Would remove ||p"
 	) &&
 	verbose test "$would_clean" = ../src/part3.c
 '
 test_expect_success C_LOCALE_OUTPUT 'git clean with absolute path' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	would_clean=$(
 		cd docs &&
 		git clean -n "$(pwd)/../src" |
 		sed -n -e "s|^Would remove ||p"
 	) &&
 	verbose test "$would_clean" = ../src/part3.c
 '
 test_expect_success 'git clean with out of work tree relative path' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	(
 		cd docs &&
 		test_must_fail git clean -n ../..
 	)
 '
 test_expect_success 'git clean with out of work tree absolute path' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	dd=$(cd .. && pwd) &&
 	(
 		cd docs &&
 		test_must_fail git clean -n $dd
 	)
 '
 test_expect_success 'git clean -d with prefix and path' '
 	mkdir -p build docs src/feature &&
 	touch a.out src/part3.c src/feature/file.c docs/manual.txt obj.o build/lib.so &&
 	(cd src/ && git clean -d feature/) &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test -f src/part3.c &&
 	test ! -f src/feature/file.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success SYMLINKS 'git clean symbolic link' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	ln -s docs/manual.txt src/part4.c &&
 	git clean &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test ! -f src/part3.c &&
 	test ! -f src/part4.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean with wildcard' '
 	touch a.clean b.clean other.c &&
 	git clean "*.clean" &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.clean &&
 	test ! -f b.clean &&
 	test -f other.c
 '
 test_expect_success 'git clean -n' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -n &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean -d' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -d &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test ! -f src/part3.c &&
 	test ! -d docs &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean -d src/ examples/' '
 	mkdir -p build docs examples &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so examples/1.c &&
 	git clean -d src/ examples/ &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test ! -f src/part3.c &&
 	test ! -f examples/1.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean -x' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -x &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test ! -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test ! -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean -d -x' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -d -x &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test ! -f src/part3.c &&
 	test ! -d docs &&
 	test ! -f obj.o &&
 	test ! -d build
 '
 test_expect_success 'git clean -d -x with ignored tracked directory' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -d -x -e src &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test -f src/part3.c &&
 	test ! -d docs &&
 	test ! -f obj.o &&
 	test ! -d build
 '
 test_expect_success 'git clean -X' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -X &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test ! -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'git clean -d -X' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -d -X &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test ! -f obj.o &&
 	test ! -d build
 '
 test_expect_success 'git clean -d -X with ignored tracked directory' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -d -X -e src &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test ! -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test ! -f obj.o &&
 	test ! -d build
 '
 test_expect_success 'clean.requireForce defaults to true' '
 	git config --unset clean.requireForce &&
 	test_must_fail git clean
 '
 test_expect_success 'clean.requireForce' '
 	git config clean.requireForce true &&
 	test_must_fail git clean
 '
 test_expect_success 'clean.requireForce and -n' '
 	mkdir -p build docs &&
 	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
 	git clean -n &&
 	test -f Makefile &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test -f a.out &&
 	test -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success 'clean.requireForce and -f' '
 	git clean -f &&
 	test -f README &&
 	test -f src/part1.c &&
 	test -f src/part2.c &&
 	test ! -f a.out &&
 	test ! -f src/part3.c &&
 	test -f docs/manual.txt &&
 	test -f obj.o &&
 	test -f build/lib.so
 '
 test_expect_success C_LOCALE_OUTPUT 'core.excludesfile' '
 	echo excludes >excludes &&
 	echo included >included &&
 	git config core.excludesfile excludes &&
 	output=$(git clean -n excludes included 2>&1) &&
 	expr "$output" : ".*included" >/dev/null &&
 	! expr "$output" : ".*excludes" >/dev/null
 '
 test_expect_success SANITY 'removal failure' '
 	mkdir foo &&
 	touch foo/bar &&
 	test_when_finished "chmod 755 foo" &&
 	(exec <foo/bar &&
 	 chmod 0 foo &&
 	 test_must_fail git clean -f -d)
 '
 test_expect_success 'nested git work tree' '
 	rm -fr foo bar baz &&
 	mkdir -p foo bar baz/boo &&
 	(
 		cd foo &&
 		git init &&
 		>hello.world
 		git add . &&
 		git commit -a -m nested
 	) &&
 	(
 		cd bar &&
 		>goodbye.people
 	) &&
 	(
 		cd baz/boo &&
 		git init &&
 		>deeper.world
 		git add . &&
 		git commit -a -m deeply.nested
 	) &&
 	git clean -f -d &&
 	test -f foo/.git/index &&
 	test -f foo/hello.world &&
 	test -f baz/boo/.git/index &&
 	test -f baz/boo/deeper.world &&
 	! test -d bar
 '
 test_expect_success 'should clean things that almost look like git but are not' '
 	rm -fr almost_git almost_bare_git almost_submodule &&
 	mkdir -p almost_git/.git/objects &&
 	mkdir -p almost_git/.git/refs &&
 	cat >almost_git/.git/HEAD <<-\EOF &&
 	garbage
 	EOF
 	cp -r almost_git/.git/ almost_bare_git &&
 	mkdir almost_submodule/ &&
 	cat >almost_submodule/.git <<-\EOF &&
 	garbage
 	EOF
 	test_when_finished "rm -rf almost_*" &&
 	git clean -f -d &&
 	test_path_is_missing almost_git &&
 	test_path_is_missing almost_bare_git &&
 	test_path_is_missing almost_submodule
 '
 test_expect_success 'should not clean submodules' '
 	rm -fr repo to_clean sub1 sub2 &&
 	mkdir repo to_clean &&
 	(
 		cd repo &&
 		git init &&
 		test_commit msg hello.world
 	) &&
 	git submodule add ./repo/.git sub1 &&
 	git commit -m "sub1" &&
 	git branch before_sub2 &&
 	git submodule add ./repo/.git sub2 &&
 	git commit -m "sub2" &&
 	git checkout before_sub2 &&
 	>to_clean/should_clean.this &&
 	git clean -f -d &&
 	test_path_is_file repo/.git/index &&
 	test_path_is_file repo/hello.world &&
 	test_path_is_file sub1/.git &&
 	test_path_is_file sub1/hello.world &&
 	test_path_is_file sub2/.git &&
 	test_path_is_file sub2/hello.world &&
 	test_path_is_missing to_clean
 '
 test_expect_success 'should avoid cleaning possible submodules' '
 	rm -fr to_clean possible_sub1 &&
 	mkdir to_clean possible_sub1 &&
 	test_when_finished "rm -rf possible_sub*" &&
 	echo "gitdir: foo" >possible_sub1/.git &&
 	>possible_sub1/hello.world &&
 	chmod 0 possible_sub1/.git &&
 	>to_clean/should_clean.this &&
 	git clean -f -d &&
 	test_path_is_file possible_sub1/.git &&
 	test_path_is_file possible_sub1/hello.world &&
 	test_path_is_missing to_clean
 '
 test_expect_success 'nested (empty) git should be kept' '
 	rm -fr empty_repo to_clean &&
 	git init empty_repo &&
 	mkdir to_clean &&
 	>to_clean/should_clean.this &&
 	git clean -f -d &&
 	test_path_is_file empty_repo/.git/HEAD &&
 	test_path_is_missing to_clean
 '
 test_expect_success 'nested bare repositories should be cleaned' '
 	rm -fr bare1 bare2 subdir &&
 	git init --bare bare1 &&
 	git clone --local --bare . bare2 &&
 	mkdir subdir &&
 	cp -r bare2 subdir/bare3 &&
 	git clean -f -d &&
 	test_path_is_missing bare1 &&
 	test_path_is_missing bare2 &&
 	test_path_is_missing subdir
 '
 test_expect_failure 'nested (empty) bare repositories should be cleaned even when in .git' '
 	rm -fr strange_bare &&
 	mkdir strange_bare &&
 	git init --bare strange_bare/.git &&
 	git clean -f -d &&
 	test_path_is_missing strange_bare
 '
 test_expect_failure 'nested (non-empty) bare repositories should be cleaned even when in .git' '
 	rm -fr strange_bare &&
 	mkdir strange_bare &&
 	git clone --local --bare . strange_bare/.git &&
 	git clean -f -d &&
 	test_path_is_missing strange_bare
 '
 test_expect_success 'giving path in nested git work tree will remove it' '
 	rm -fr repo &&
 	mkdir repo &&
 	(
 		cd repo &&
 		git init &&
 		mkdir -p bar/baz &&
 		test_commit msg bar/baz/hello.world
 	) &&
 	git clean -f -d repo/bar/baz &&
 	test_path_is_file repo/.git/HEAD &&
 	test_path_is_dir repo/bar/ &&
 	test_path_is_missing repo/bar/baz
 '
 test_expect_success 'giving path to nested .git will not remove it' '
 	rm -fr repo &&
 	mkdir repo untracked &&
 	(
 		cd repo &&
 		git init &&
 		test_commit msg hello.world
 	) &&
 	git clean -f -d repo/.git &&
 	test_path_is_file repo/.git/HEAD &&
 	test_path_is_dir repo/.git/refs &&
 	test_path_is_dir repo/.git/objects &&
 	test_path_is_dir untracked/
 '
 test_expect_success 'giving path to nested .git/ will remove contents' '
 	rm -fr repo untracked &&
 	mkdir repo untracked &&
 	(
 		cd repo &&
 		git init &&
 		test_commit msg hello.world
 	) &&
 	git clean -f -d repo/.git/ &&
 	test_path_is_dir repo/.git &&
 	test_dir_is_empty repo/.git &&
 	test_path_is_dir untracked/
 '
 test_expect_success 'force removal of nested git work tree' '
 	rm -fr foo bar baz &&
 	mkdir -p foo bar baz/boo &&
 	(
 		cd foo &&
 		git init &&
 		>hello.world
 		git add . &&
 		git commit -a -m nested
 	) &&
 	(
 		cd bar &&
 		>goodbye.people
 	) &&
 	(
 		cd baz/boo &&
 		git init &&
 		>deeper.world
 		git add . &&
 		git commit -a -m deeply.nested
 	) &&
 	git clean -f -f -d &&
 	! test -d foo &&
 	! test -d bar &&
 	! test -d baz
 '
 test_expect_success 'git clean -e' '
 	rm -fr repo &&
 	mkdir repo &&
 	(
 		cd repo &&
 		git init &&
 		touch known 1 2 3 &&
 		git add known &&
 		git clean -f -e 1 -e 2 &&
 		test -e 1 &&
 		test -e 2 &&
 		! (test -e 3) &&
 		test -e known
 	)
 '
 test_expect_success SANITY 'git clean -d with an unreadable empty directory' '
 	mkdir foo &&
 	chmod a= foo &&
 	git clean -dfx foo &&
 	! test -d foo
 '
 test_expect_success 'git clean -d respects pathspecs (dir is prefix of pathspec)' '
 	mkdir -p foo &&
 	mkdir -p foobar &&
 	git clean -df foobar &&
 	test_path_is_dir foo &&
 	test_path_is_missing foobar
 '
 test_expect_success 'git clean -d respects pathspecs (pathspec is prefix of dir)' '
 	mkdir -p foo &&
 	mkdir -p foobar &&
 	git clean -df foo &&
 	test_path_is_missing foo &&
 	test_path_is_dir foobar
 '
 test_done