mirror of
https://github.com/git/git.git
synced 2024-10-30 22:07:53 +01:00
9035d75a2b
The data is streamed to the filter process anyway. Better avoid mapping the file if possible. This is especially useful if a clean filter reduces the size, for example if it computes a sha1 for binary data, like git media. The file size that the previous implementation could handle was limited by the available address space; large files for example could not be handled with (32-bit) msysgit. The new implementation can filter files of any size as long as the filter output is small enough. The new code path is only taken if the filter is required. The filter consumes data directly from the fd. If it fails, the original data is not immediately available. The condition can easily be handled as a fatal error, which is expected for a required filter anyway. If the filter was not required, the condition would need to be handled in a different way, like seeking to 0 and reading the data. But this would require more restructuring of the code and is probably not worth it. The obvious approach of falling back to reading all data would not help achieving the main purpose of this patch, which is to handle large files with limited address space. If reading all data is an option, we can simply take the old code path right away and mmap the entire file. The environment variable GIT_MMAP_LIMIT, which has been introduced in a previous commit is used to test that the expected code path is taken. A related test that exercises required filters is modified to verify that the data actually has been modified on its way from the file system to the object store. Signed-off-by: Steffen Prohaska <prohaska@zib.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
81 lines
2.3 KiB
C
81 lines
2.3 KiB
C
/*
|
|
* Copyright (c) 2011, Google Inc.
|
|
*/
|
|
#ifndef CONVERT_H
|
|
#define CONVERT_H
|
|
|
|
enum safe_crlf {
|
|
SAFE_CRLF_FALSE = 0,
|
|
SAFE_CRLF_FAIL = 1,
|
|
SAFE_CRLF_WARN = 2
|
|
};
|
|
|
|
extern enum safe_crlf safe_crlf;
|
|
|
|
enum auto_crlf {
|
|
AUTO_CRLF_FALSE = 0,
|
|
AUTO_CRLF_TRUE = 1,
|
|
AUTO_CRLF_INPUT = -1
|
|
};
|
|
|
|
extern enum auto_crlf auto_crlf;
|
|
|
|
enum eol {
|
|
EOL_UNSET,
|
|
EOL_CRLF,
|
|
EOL_LF,
|
|
#ifdef NATIVE_CRLF
|
|
EOL_NATIVE = EOL_CRLF
|
|
#else
|
|
EOL_NATIVE = EOL_LF
|
|
#endif
|
|
};
|
|
|
|
extern enum eol core_eol;
|
|
|
|
/* returns 1 if *dst was used */
|
|
extern int convert_to_git(const char *path, const char *src, size_t len,
|
|
struct strbuf *dst, enum safe_crlf checksafe);
|
|
extern int convert_to_working_tree(const char *path, const char *src,
|
|
size_t len, struct strbuf *dst);
|
|
extern int renormalize_buffer(const char *path, const char *src, size_t len,
|
|
struct strbuf *dst);
|
|
static inline int would_convert_to_git(const char *path)
|
|
{
|
|
return convert_to_git(path, NULL, 0, NULL, 0);
|
|
}
|
|
/* Precondition: would_convert_to_git_filter_fd(path) == true */
|
|
extern void convert_to_git_filter_fd(const char *path, int fd,
|
|
struct strbuf *dst,
|
|
enum safe_crlf checksafe);
|
|
extern int would_convert_to_git_filter_fd(const char *path);
|
|
|
|
/*****************************************************************
|
|
*
|
|
* Streaming conversion support
|
|
*
|
|
*****************************************************************/
|
|
|
|
struct stream_filter; /* opaque */
|
|
|
|
extern struct stream_filter *get_stream_filter(const char *path, const unsigned char *);
|
|
extern void free_stream_filter(struct stream_filter *);
|
|
extern int is_null_stream_filter(struct stream_filter *);
|
|
|
|
/*
|
|
* Use as much input up to *isize_p and fill output up to *osize_p;
|
|
* update isize_p and osize_p to indicate how much buffer space was
|
|
* consumed and filled. Return 0 on success, non-zero on error.
|
|
*
|
|
* Some filters may need to buffer the input and look-ahead inside it
|
|
* to decide what to output, and they may consume more than zero bytes
|
|
* of input and still not produce any output. After feeding all the
|
|
* input, pass NULL as input and keep calling this function, to let
|
|
* such filters know there is no more input coming and it is time for
|
|
* them to produce the remaining output based on the buffered input.
|
|
*/
|
|
extern int stream_filter(struct stream_filter *,
|
|
const char *input, size_t *isize_p,
|
|
char *output, size_t *osize_p);
|
|
|
|
#endif /* CONVERT_H */
|