Removed buffer size increase. Only tells the filesize to Windows now. On 26.9.2014 14:08, Erik de Castro Lopo wrote:> Martijn van Beurden wrote: > >> Can you please wrap the setvbuf in _WIN32 IFDEFs too? Currently >> memory usage of FLAC decoding is about 1MB, so this patch is >> increasing memory usage tenfold, also for platforms that do not >> need this. It is a non-problem on my system anyway. > +1 > > Will apply patch when it (and the previous one) are wrapped in > Win32 #ifdef or similar. > > Erik-------------- next part -------------- diff --git a/src/flac/decode.c b/src/flac/decode.c index 5e5e17a..7705107 100644 --- a/src/flac/decode.c +++ b/src/flac/decode.c @@ -101,7 +101,7 @@ static FLAC__bool is_big_endian_host_; static FLAC__bool DecoderSession_construct(DecoderSession *d, FLAC__bool is_ogg, FLAC__bool use_first_serial_number, long serial_number, FileFormat format, FLAC__bool treat_warnings_as_errors, FLAC__bool continue_through_decode_errors, FLAC__bool channel_map_none, replaygain_synthesis_spec_t replaygain_synthesis_spec, FLAC__bool analysis_mode, analysis_options aopts, utils__SkipUntilSpecification *skip_specification, utils__SkipUntilSpecification *until_specification, utils__CueSpecification *cue_specification, foreign_metadata_t *foreign_metadata, const char *infilename, const char *outfilename); static void DecoderSession_destroy(DecoderSession *d, FLAC__bool error_occurred); static FLAC__bool DecoderSession_init_decoder(DecoderSession *d, const char *infilename); -static FLAC__bool DecoderSession_process(DecoderSession *d); +static FLAC__bool DecoderSession_process(DecoderSession *d, const char *outfilename); static int DecoderSession_finish_ok(DecoderSession *d); static int DecoderSession_finish_error(DecoderSession *d); static FLAC__bool canonicalize_until_specification(utils__SkipUntilSpecification *spec, const char *inbasefilename, unsigned sample_rate, FLAC__uint64 skip, FLAC__uint64 total_samples_in_input); @@ -177,7 +177,7 @@ int flac__decode_file(const char *infilename, const char *outfilename, FLAC__boo if(!DecoderSession_init_decoder(&decoder_session, infilename)) return DecoderSession_finish_error(&decoder_session); - if(!DecoderSession_process(&decoder_session)) + if(!DecoderSession_process(&decoder_session, outfilename)) return DecoderSession_finish_error(&decoder_session); return DecoderSession_finish_ok(&decoder_session); @@ -317,7 +317,7 @@ FLAC__bool DecoderSession_init_decoder(DecoderSession *decoder_session, const ch return true; } -FLAC__bool DecoderSession_process(DecoderSession *d) +FLAC__bool DecoderSession_process(DecoderSession *d, const char *outfilename) { if(!FLAC__stream_decoder_process_until_end_of_metadata(d->decoder)) { flac__utils_printf(stderr, 2, "\n"); @@ -366,6 +366,28 @@ FLAC__bool DecoderSession_process(DecoderSession *d) /* write the WAVE/AIFF headers if necessary */ if(!d->analysis_mode && !d->test_only && d->format != FORMAT_RAW) { +#ifdef _WIN32 + if(strcmp(outfilename, "-") && d->total_samples > 0) { + HANDLE fh = CreateFile_utf8(outfilename, GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if(fh != INVALID_HANDLE_VALUE) { + if (GetFileType(fh) == FILE_TYPE_DISK) { + LARGE_INTEGER size; + size.QuadPart = d->total_samples * d->channels * ((d->bps+7)/8) + 512; + if(d->foreign_metadata) { + size_t i; + for(i = d->format==FORMAT_RF64?2:1; i < d->foreign_metadata->num_blocks; i++) { + if(i != d->foreign_metadata->format_block && i != d->foreign_metadata->audio_block) + size.QuadPart += d->foreign_metadata->blocks[i].size; + } + } + + if(SetFilePointerEx(fh, size, NULL, FILE_CURRENT)) /* tell filesystem the expected filesize to eliminate fragmentation */ + SetEndOfFile(fh); + } + CloseHandle(fh); + } + } +#endif if(!write_iff_headers(d->fout, d, d->total_samples)) { d->abort_flag = true; return false;
Erik de Castro Lopo
2014-Sep-27 05:54 UTC
[flac-dev] Patch to add buffering to decoding too
Janne Hyv?rinen wrote:> Removed buffer size increase. Only tells the filesize to Windows now.Would it not be possible to do the same when encoding? Of course we don't know the exact size of the output file, but guessing at 70% of 80% of the input file size would be correct in most cases? Are there any negative consequences of overestimating like this? Erik -- ---------------------------------------------------------------------- Erik de Castro Lopo http://www.mega-nerd.com/
On 27.9.2014 8:54, Erik de Castro Lopo wrote:> Janne Hyv?rinen wrote: > >> Removed buffer size increase. Only tells the filesize to Windows now. > Would it not be possible to do the same when encoding? Of course we > don't know the exact size of the output file, but guessing at 70% of > 80% of the input file size would be correct in most cases? Are there > any negative consequences of overestimating like this? >Estimating the size too large while other processes are writing to the drive at the same time would leave a gap behind the file. My fear was that writing thousands of files one after another, and many of them leaving these empty spaces, would eventually cause the drive to be sparsely filled resulting in all further large file writes to being spread all around the drive. Underestimating a little would of course not cause this.
The previous patch was bugged. The output file wasn't truncated to
correct size and was a bit off from rounding the WAVE/AIFF header to
smallest sector size. And RAW output didn't benefit from the change. And
the existing functions didn't need changes as outputfilename was already
known. Attached is a fixed and improved version.
-------------- next part --------------
diff --git a/src/flac/decode.c b/src/flac/decode.c
index 5e5e17a..ed7eff8 100644
--- a/src/flac/decode.c
+++ b/src/flac/decode.c
@@ -263,6 +263,23 @@ FLAC__bool DecoderSession_construct(DecoderSession *d,
FLAC__bool is_ogg, FLAC__
void DecoderSession_destroy(DecoderSession *d, FLAC__bool error_occurred)
{
if(0 != d->fout && d->fout != stdout) {
+#ifdef _WIN32
+ if(!error_occurred) {
+ FLAC__off_t written_size = ftello(d->fout);
+ if(written_size > 0) {
+ HANDLE fh = CreateFile_utf8(d->outfilename, GENERIC_READ|GENERIC_WRITE,
FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
NULL);
+ if(fh != INVALID_HANDLE_VALUE) {
+ if(GetFileType(fh) == FILE_TYPE_DISK) {
+ LARGE_INTEGER size;
+ size.QuadPart = written_size;
+ if(SetFilePointerEx(fh, size, NULL, FILE_CURRENT)) /* correct the file
size */
+ SetEndOfFile(fh);
+ }
+ CloseHandle(fh);
+ }
+ }
+ }
+#endif
fclose(d->fout);
if(error_occurred)
flac_unlink(d->outfilename);
@@ -364,6 +381,32 @@ FLAC__bool DecoderSession_process(DecoderSession *d)
}
}
+#ifdef _WIN32
+ if(!d->analysis_mode && !d->test_only &&
d->total_samples > 0 && d->fout != stdout) {
+ HANDLE fh = CreateFile_utf8(d->outfilename, GENERIC_READ|GENERIC_WRITE,
FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
NULL);
+ if(fh != INVALID_HANDLE_VALUE) {
+ if (GetFileType(fh) == FILE_TYPE_DISK) {
+ LARGE_INTEGER size;
+ size.QuadPart = d->total_samples * d->channels * ((d->bps+7)/8);
+ if(d->format != FORMAT_RAW) {
+ size.QuadPart += 512;
+ if(d->foreign_metadata) {
+ size_t i;
+ for(i = d->format==FORMAT_RF64?2:1; i <
d->foreign_metadata->num_blocks; i++) {
+ if(i != d->foreign_metadata->format_block && i !=
d->foreign_metadata->audio_block)
+ size.QuadPart += d->foreign_metadata->blocks[i].size;
+ }
+ }
+ }
+
+ if(SetFilePointerEx(fh, size, NULL, FILE_CURRENT)) /* tell filesystem the
expected filesize to eliminate fragmentation */
+ SetEndOfFile(fh);
+ }
+ CloseHandle(fh);
+ }
+ }
+#endif
+
/* write the WAVE/AIFF headers if necessary */
if(!d->analysis_mode && !d->test_only && d->format !=
FORMAT_RAW) {
if(!write_iff_headers(d->fout, d, d->total_samples)) {
Erik de Castro Lopo
2014-Oct-02 20:52 UTC
[flac-dev] Patch to add buffering to decoding too
Janne Hyv?rinen wrote:> The previous patch was bugged. The output file wasn't truncated to > correct size and was a bit off from rounding the WAVE/AIFF header to > smallest sector size. And RAW output didn't benefit from the change. And > the existing functions didn't need changes as outputfilename was already > known. Attached is a fixed and improved version.Applied. Thanks. Erik -- ---------------------------------------------------------------------- Erik de Castro Lopo http://www.mega-nerd.com/