flac dev - Sep 2014 - Patch to add buffering to decoding too

Decoding flac files is also prone to producing fragmented files. NTFS 
has the ability to completely avoid fragmentation if it is told the file 
size before hand, but that would require using special Windows-only 
functions. Increasing the write buffer from the default 512 bytes to 10 
MB already reduces the problem tremendously.

-------------- next part --------------
diff --git a/src/flac/decode.c b/src/flac/decode.c
index 5e5e17a..9e9405c 100644
--- a/src/flac/decode.c
+++ b/src/flac/decode.c
@@ -251,6 +251,7 @@ FLAC__bool DecoderSession_construct(DecoderSession *d,
FLAC__bool is_ogg, FLAC__
 				DecoderSession_destroy(d, /*error_occurred=*/true);
 				return false;
 			}
+			setvbuf(d->fout, NULL, _IOFBF, 10*1024*1024); /* 10MB output buffer to
help reduce disk fragmentation */
 		}
 	}

I made some changes to the previous patch. I don't know why I originally 
didn't put the output buffering to piped output too but that is now 
moved to cover both file and pipe output.
Additionally this patch informs the Windows filesystem in advance about 
the decoded size to eliminate NTFS fragmentation.

On 25.9.2014 23:01, Janne Hyv?rinen wrote:
> Decoding flac files is also prone to producing fragmented files. NTFS 
> has the ability to completely avoid fragmentation if it is told the 
> file size before hand, but that would require using special 
> Windows-only functions. Increasing the write buffer from the default 
> 512 bytes to 10 MB already reduces the problem tremendously.
>
>
>
> _______________________________________________
> flac-dev mailing list
> flac-dev at xiph.org
> http://lists.xiph.org/mailman/listinfo/flac-dev
-------------- next part --------------
An HTML attachment was scrubbed...
URL:
http://lists.xiph.org/pipermail/flac-dev/attachments/20140926/a0386298/attachment.htm
-------------- next part --------------
diff --git a/src/flac/decode.c b/src/flac/decode.c
index 5e5e17a..90e0b1a 100644
--- a/src/flac/decode.c
+++ b/src/flac/decode.c
@@ -101,7 +101,7 @@ static FLAC__bool is_big_endian_host_;
 static FLAC__bool DecoderSession_construct(DecoderSession *d, FLAC__bool
is_ogg, FLAC__bool use_first_serial_number, long serial_number, FileFormat
format, FLAC__bool treat_warnings_as_errors, FLAC__bool
continue_through_decode_errors, FLAC__bool channel_map_none,
replaygain_synthesis_spec_t replaygain_synthesis_spec, FLAC__bool analysis_mode,
analysis_options aopts, utils__SkipUntilSpecification *skip_specification,
utils__SkipUntilSpecification *until_specification, utils__CueSpecification
*cue_specification, foreign_metadata_t *foreign_metadata, const char
*infilename, const char *outfilename);
 static void DecoderSession_destroy(DecoderSession *d, FLAC__bool
error_occurred);
 static FLAC__bool DecoderSession_init_decoder(DecoderSession *d, const char
*infilename);
-static FLAC__bool DecoderSession_process(DecoderSession *d);
+static FLAC__bool DecoderSession_process(DecoderSession *d, const char
*outfilename);
 static int DecoderSession_finish_ok(DecoderSession *d);
 static int DecoderSession_finish_error(DecoderSession *d);
 static FLAC__bool
canonicalize_until_specification(utils__SkipUntilSpecification *spec, const char
*inbasefilename, unsigned sample_rate, FLAC__uint64 skip, FLAC__uint64
total_samples_in_input);
@@ -177,7 +177,7 @@ int flac__decode_file(const char *infilename, const char
*outfilename, FLAC__boo
 	if(!DecoderSession_init_decoder(&decoder_session, infilename))
 		return DecoderSession_finish_error(&decoder_session);
 
-	if(!DecoderSession_process(&decoder_session))
+	if(!DecoderSession_process(&decoder_session, outfilename))
 		return DecoderSession_finish_error(&decoder_session);
 
 	return DecoderSession_finish_ok(&decoder_session);
@@ -252,6 +252,7 @@ FLAC__bool DecoderSession_construct(DecoderSession *d,
FLAC__bool is_ogg, FLAC__
 				return false;
 			}
 		}
+		setvbuf(d->fout, NULL, _IOFBF, 10*1024*1024); /* 10MB output buffer to
help reduce disk fragmentation */
 	}
 
 	if(analysis_mode)
@@ -317,7 +318,7 @@ FLAC__bool DecoderSession_init_decoder(DecoderSession
*decoder_session, const ch
 	return true;
 }
 
-FLAC__bool DecoderSession_process(DecoderSession *d)
+FLAC__bool DecoderSession_process(DecoderSession *d, const char *outfilename)
 {
 	if(!FLAC__stream_decoder_process_until_end_of_metadata(d->decoder)) {
 		flac__utils_printf(stderr, 2, "\n");
@@ -366,6 +367,29 @@ FLAC__bool DecoderSession_process(DecoderSession *d)
 
 	/* write the WAVE/AIFF headers if necessary */
 	if(!d->analysis_mode && !d->test_only && d->format !=
FORMAT_RAW) {
+#ifdef _WIN32
+		if(strcmp(outfilename, "-") && d->total_samples > 0)
{
+			HANDLE fh = CreateFile_utf8(outfilename, GENERIC_READ|GENERIC_WRITE,
FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
NULL);
+			if(fh != INVALID_HANDLE_VALUE) {
+				if (GetFileType(fh) == FILE_TYPE_DISK) {
+					LARGE_INTEGER size, pos;
+
+					size.QuadPart = d->total_samples * d->channels * ((d->bps+7)/8) +
512;
+					if(d->foreign_metadata) {
+						size_t i;
+						for(i = d->format==FORMAT_RF64?2:1; i <
d->foreign_metadata->num_blocks; i++) {
+							if(i != d->foreign_metadata->format_block && i !=
d->foreign_metadata->audio_block)
+								size.QuadPart += d->foreign_metadata->blocks[i].size;
+						}
+					}
+
+					if(SetFilePointerEx(fh, size, NULL, FILE_CURRENT)) /* tell filesystem the
expected filesize to eliminate fragmentation */
+						SetEndOfFile(fh); 
+				}
+				CloseHandle(fh);
+			}
+		}
+#endif
 		if(!write_iff_headers(d->fout, d, d->total_samples)) {
 			d->abort_flag = true;
 			return false;

Can you please wrap the setvbuf in _WIN32 IFDEFs too? Currently 
memory usage of FLAC decoding is about 1MB, so this patch is 
increasing memory usage tenfold, also for platforms that do not 
need this. It is a non-problem on my system anyway.

Op 26-09-14 om 10:36 schreef Janne Hyv?rinen:
> I made some changes to the previous patch. I don't know why I 
> originally didn't put the output buffering to piped output too 
> but that is now moved to cover both file and pipe output.
> Additionally this patch informs the Windows filesystem in 
> advance about the decoded size to eliminate NTFS fragmentation.
>
> On 25.9.2014 23:01, Janne Hyv?rinen wrote:
>> Decoding flac files is also prone to producing fragmented 
>> files. NTFS has the ability to completely avoid fragmentation 
>> if it is told the file size before hand, but that would 
>> require using special Windows-only functions. Increasing the 
>> write buffer from the default 512 bytes to 10 MB already 
>> reduces the problem tremendously.
>>
>>
>>
>> _______________________________________________
>> flac-dev mailing list
>> flac-dev at xiph.org
>> http://lists.xiph.org/mailman/listinfo/flac-dev
>
>
>
> _______________________________________________
> flac-dev mailing list
> flac-dev at xiph.org
> http://lists.xiph.org/mailman/listinfo/flac-dev
-------------- next part --------------
An HTML attachment was scrubbed...
URL:
http://lists.xiph.org/pipermail/flac-dev/attachments/20140926/b2162d36/attachment.htm