Decoding flac files is also prone to producing fragmented files. NTFS has the ability to completely avoid fragmentation if it is told the file size before hand, but that would require using special Windows-only functions. Increasing the write buffer from the default 512 bytes to 10 MB already reduces the problem tremendously. -------------- next part -------------- diff --git a/src/flac/decode.c b/src/flac/decode.c index 5e5e17a..9e9405c 100644 --- a/src/flac/decode.c +++ b/src/flac/decode.c @@ -251,6 +251,7 @@ FLAC__bool DecoderSession_construct(DecoderSession *d, FLAC__bool is_ogg, FLAC__ DecoderSession_destroy(d, /*error_occurred=*/true); return false; } + setvbuf(d->fout, NULL, _IOFBF, 10*1024*1024); /* 10MB output buffer to help reduce disk fragmentation */ } }
I made some changes to the previous patch. I don't know why I originally didn't put the output buffering to piped output too but that is now moved to cover both file and pipe output. Additionally this patch informs the Windows filesystem in advance about the decoded size to eliminate NTFS fragmentation. On 25.9.2014 23:01, Janne Hyv?rinen wrote:> Decoding flac files is also prone to producing fragmented files. NTFS > has the ability to completely avoid fragmentation if it is told the > file size before hand, but that would require using special > Windows-only functions. Increasing the write buffer from the default > 512 bytes to 10 MB already reduces the problem tremendously. > > > > _______________________________________________ > flac-dev mailing list > flac-dev at xiph.org > http://lists.xiph.org/mailman/listinfo/flac-dev-------------- next part -------------- An HTML attachment was scrubbed... URL: http://lists.xiph.org/pipermail/flac-dev/attachments/20140926/a0386298/attachment.htm -------------- next part -------------- diff --git a/src/flac/decode.c b/src/flac/decode.c index 5e5e17a..90e0b1a 100644 --- a/src/flac/decode.c +++ b/src/flac/decode.c @@ -101,7 +101,7 @@ static FLAC__bool is_big_endian_host_; static FLAC__bool DecoderSession_construct(DecoderSession *d, FLAC__bool is_ogg, FLAC__bool use_first_serial_number, long serial_number, FileFormat format, FLAC__bool treat_warnings_as_errors, FLAC__bool continue_through_decode_errors, FLAC__bool channel_map_none, replaygain_synthesis_spec_t replaygain_synthesis_spec, FLAC__bool analysis_mode, analysis_options aopts, utils__SkipUntilSpecification *skip_specification, utils__SkipUntilSpecification *until_specification, utils__CueSpecification *cue_specification, foreign_metadata_t *foreign_metadata, const char *infilename, const char *outfilename); static void DecoderSession_destroy(DecoderSession *d, FLAC__bool error_occurred); static FLAC__bool DecoderSession_init_decoder(DecoderSession *d, const char *infilename); -static FLAC__bool DecoderSession_process(DecoderSession *d); +static FLAC__bool DecoderSession_process(DecoderSession *d, const char *outfilename); static int DecoderSession_finish_ok(DecoderSession *d); static int DecoderSession_finish_error(DecoderSession *d); static FLAC__bool canonicalize_until_specification(utils__SkipUntilSpecification *spec, const char *inbasefilename, unsigned sample_rate, FLAC__uint64 skip, FLAC__uint64 total_samples_in_input); @@ -177,7 +177,7 @@ int flac__decode_file(const char *infilename, const char *outfilename, FLAC__boo if(!DecoderSession_init_decoder(&decoder_session, infilename)) return DecoderSession_finish_error(&decoder_session); - if(!DecoderSession_process(&decoder_session)) + if(!DecoderSession_process(&decoder_session, outfilename)) return DecoderSession_finish_error(&decoder_session); return DecoderSession_finish_ok(&decoder_session); @@ -252,6 +252,7 @@ FLAC__bool DecoderSession_construct(DecoderSession *d, FLAC__bool is_ogg, FLAC__ return false; } } + setvbuf(d->fout, NULL, _IOFBF, 10*1024*1024); /* 10MB output buffer to help reduce disk fragmentation */ } if(analysis_mode) @@ -317,7 +318,7 @@ FLAC__bool DecoderSession_init_decoder(DecoderSession *decoder_session, const ch return true; } -FLAC__bool DecoderSession_process(DecoderSession *d) +FLAC__bool DecoderSession_process(DecoderSession *d, const char *outfilename) { if(!FLAC__stream_decoder_process_until_end_of_metadata(d->decoder)) { flac__utils_printf(stderr, 2, "\n"); @@ -366,6 +367,29 @@ FLAC__bool DecoderSession_process(DecoderSession *d) /* write the WAVE/AIFF headers if necessary */ if(!d->analysis_mode && !d->test_only && d->format != FORMAT_RAW) { +#ifdef _WIN32 + if(strcmp(outfilename, "-") && d->total_samples > 0) { + HANDLE fh = CreateFile_utf8(outfilename, GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if(fh != INVALID_HANDLE_VALUE) { + if (GetFileType(fh) == FILE_TYPE_DISK) { + LARGE_INTEGER size, pos; + + size.QuadPart = d->total_samples * d->channels * ((d->bps+7)/8) + 512; + if(d->foreign_metadata) { + size_t i; + for(i = d->format==FORMAT_RF64?2:1; i < d->foreign_metadata->num_blocks; i++) { + if(i != d->foreign_metadata->format_block && i != d->foreign_metadata->audio_block) + size.QuadPart += d->foreign_metadata->blocks[i].size; + } + } + + if(SetFilePointerEx(fh, size, NULL, FILE_CURRENT)) /* tell filesystem the expected filesize to eliminate fragmentation */ + SetEndOfFile(fh); + } + CloseHandle(fh); + } + } +#endif if(!write_iff_headers(d->fout, d, d->total_samples)) { d->abort_flag = true; return false;
Martijn van Beurden
2014-Sep-26 10:09 UTC
[flac-dev] Patch to add buffering to decoding too
Can you please wrap the setvbuf in _WIN32 IFDEFs too? Currently memory usage of FLAC decoding is about 1MB, so this patch is increasing memory usage tenfold, also for platforms that do not need this. It is a non-problem on my system anyway. Op 26-09-14 om 10:36 schreef Janne Hyv?rinen:> I made some changes to the previous patch. I don't know why I > originally didn't put the output buffering to piped output too > but that is now moved to cover both file and pipe output. > Additionally this patch informs the Windows filesystem in > advance about the decoded size to eliminate NTFS fragmentation. > > On 25.9.2014 23:01, Janne Hyv?rinen wrote: >> Decoding flac files is also prone to producing fragmented >> files. NTFS has the ability to completely avoid fragmentation >> if it is told the file size before hand, but that would >> require using special Windows-only functions. Increasing the >> write buffer from the default 512 bytes to 10 MB already >> reduces the problem tremendously. >> >> >> >> _______________________________________________ >> flac-dev mailing list >> flac-dev at xiph.org >> http://lists.xiph.org/mailman/listinfo/flac-dev > > > > _______________________________________________ > flac-dev mailing list > flac-dev at xiph.org > http://lists.xiph.org/mailman/listinfo/flac-dev-------------- next part -------------- An HTML attachment was scrubbed... URL: http://lists.xiph.org/pipermail/flac-dev/attachments/20140926/b2162d36/attachment.htm