Here's some stuff people might find usefull. Below is a patch to beta4
which makes res0.c output histogram files rather than coefficient files
for entropy. This means that you can run a large dataset through and only
have a few tiny file afterwards (one for each longshort mode/book).
also in there is vq/mybuild.c which generates hexagonal cell 2d codebooks.
finally vq/refinehuff.c takes an existing .vqh file and a histogram file
from the modified res0.c and only switches out the huffman lengths.
I hope this is useful to someone (even if the code is ugly as heck).
(note that this includes vqsplit.h as it is missing in beta4 but is
present in cvs).
Arnar.
Patch below
-------------
diff -N -r -c -b ../libvorbis-1.0beta4/lib/res0.c ./lib/res0.c
*** ../libvorbis-1.0beta4/lib/res0.c Sun Feb 25 23:37:59 2001
--- ./lib/res0.c Tue May 1 18:19:19 2001
***************
*** 211,216 ****
--- 211,295 ----
return(i);
}
+
+
+ #define MXMODE 2
+ #define MXPART 10
+
+ typedef struct {
+ int maxent; /* highest stored entry */
+ int *bin;
+ } histbin;
+
+ static histbin hist[MXMODE][MXPART];
+
+
+
+ /* This wont win any speed contests ... */
+
+ static
+ void dumphist(void) {
+ int m;
+ int p;
+ static int cnt = 0;
+ FILE * fp;
+ char fname[255];
+ for(m=0; m<MXMODE; m++) for(p=0; p<MXPART; p++)
+ if (hist[m][p].bin) {
+ int i;
+ sprintf(fname, "reshist_%d_%d.hst", m, p);
+ fp = fopen(fname, "a");
+ /* fprintf(fp, "%d %d\n", m, p); */
+ for(i=0;i<=hist[m][p].maxent; i++)
+ fprintf(fp, "%d, ", hist[m][p].bin[i]);
+ fprintf(fp, "\n");
+ fclose(fp);
+ }
+ }
+
+ static
+ void
+ histhack(int mode, int part, int entry) {
+ static int init = 0;
+ static int hit = 0;
+
+ if ((mode < 0) || (mode > MXMODE-1) ||
+ (part < 0) || (part > MXPART-1) ) {
+ fprintf(stderr,"OUT OF BOUNDS: mode = %d part = %d\n", mode,
part);
+ exit(2);
+ }
+
+ if (!init) {
+ int m, p;
+ for(m=0; m<MXMODE; m++) for(p=0; p<MXPART; p++) {
+ hist[m][p].maxent = -1;
+ hist[m][p].bin = NULL;
+ }
+ atexit(dumphist);
+ init++;
+ }
+
+ if (entry > hist[mode][part].maxent) {
+ int i;
+ hist[mode][part].bin = realloc(hist[mode][part].bin,
sizeof(int)*(entry+1));
+ for(i=hist[mode][part].maxent+1; i<=entry; i++) hist[mode][part].bin[i]
= 0;
+ hist[mode][part].maxent = entry;
+ }
+ hist[mode][part].bin[entry]++;
+ }
+
+
+
+ /*
+ opb - buffer
+ vec - data to quantize
+ n - samples per partition
+ stages - probably number of stages to iterate, used as a boolean for now
+ books - pointer to an array of codebooks, only the first one is used.
+ mode - long/short blocks
+ part - codebook number
+ */
+
static int _encodepart(oggpack_buffer *opb,float *vec, int n,
int stages, codebook **books,int mode,int part){
int i,j=0,bits=0;
***************
*** 219,224 ****
--- 298,304 ----
int step=n/dim;
for(i=0;i<step;i++){
int entry=vorbis_book_besterror(books[j],vec+i,step,0);
+ histhack(mode, part, entry);
#ifdef TRAIN_RESENT
{
char buf[80];
***************
*** 231,237 ****
#endif
bits+=vorbis_book_encode(books[j],entry,opb);
}
! }
return(bits);
}
--- 311,317 ----
#endif
bits+=vorbis_book_encode(books[j],entry,opb);
}
! } else histhack(mode, part, 0);
return(bits);
}
diff -N -r -c -b ../libvorbis-1.0beta4/vq/Makefile.am ./vq/Makefile.am
*** ../libvorbis-1.0beta4/vq/Makefile.am Sun Jan 21 20:38:51 2001
--- ./vq/Makefile.am Tue May 1 18:26:03 2001
***************
*** 6,12 ****
EXTRA_PROGRAMS = lspvqtrain genericvqtrain residuevqtrain vqbuild\
vqcascade vqmetrics latticebuild latticepare latticehint\
! latticetune huffbuild residuesplit distribution
CLEANFILES = $(EXTRA_PROGRAMS)
LDFLAGS = -static
--- 6,12 ----
EXTRA_PROGRAMS = lspvqtrain genericvqtrain residuevqtrain vqbuild\
vqcascade vqmetrics latticebuild latticepare latticehint\
! latticetune huffbuild residuesplit distribution mybuild refinehuff
CLEANFILES = $(EXTRA_PROGRAMS)
LDFLAGS = -static
***************
*** 20,25 ****
--- 20,29 ----
vqgen.h bookutil.h vqext.h
vqbuild_SOURCES = build.c vqgen.c bookutil.c vqsplit.c\
vqgen.h bookutil.h vqext.h
+ mybuild_SOURCES = mybuild.c vqgen.c bookutil.c vqsplit.c\
+ vqgen.h bookutil.h vqext.h
+ refinehuff_SOURCES = refinehuff.c vqgen.c bookutil.c vqsplit.c\
+ vqgen.h bookutil.h vqext.h
vqcascade_SOURCES = run.c cascade.c vqgen.c bookutil.c\
vqgen.h bookutil.h
vqmetrics_SOURCES = run.c metrics.c vqgen.c bookutil.c\
diff -N -r -c -b ../libvorbis-1.0beta4/vq/mybuild.c ./vq/mybuild.c
*** ../libvorbis-1.0beta4/vq/mybuild.c Wed Dec 31 19:00:00 1969
--- ./vq/mybuild.c Tue May 1 18:24:41 2001
***************
*** 0 ****
--- 1,178 ----
+ /********************************************************************
+ * *
+ * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
+ * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
+ * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
+ * *
+ * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
+ * by the XIPHOPHORUS Company http://www.xiph.org/ *
+
+ ********************************************************************
+
+ function: utility main for building codebooks from training sets
+ last mod: $Id: build.c,v 1.20 2001/02/26 03:51:12 xiphmont Exp $
+
+ ********************************************************************/
+
+ #include <stdlib.h>
+ #include <stdio.h>
+ #include <math.h>
+ #include <string.h>
+ #include <errno.h>
+ #include "bookutil.h"
+
+ #include "vqgen.h"
+ #include "vqsplit.h"
+
+
+ int
+ mkcode(float i, float j, float max, float *xs, float *ys) {
+ int cnt = 0;
+ float x = 0.5*(i+j);
+ float y = 0.5*(i-j)*sqrt(3);
+
+ if ((abs(x)<max) && (abs(y)<max)) cnt++;
+ if (xs) *xs = x;
+ if (ys) *ys = y;
+ return cnt;
+ }
+
+
+ int
+ make_hexlat(int levels, float max, float scale, float **vcs) {
+ int i, j;
+ float x, y;
+ int scnt, cnt = 0;
+ float *vc;
+
+ /* counting pass */
+ for(i=-levels*2; i<=levels*2; i++) {
+ for(j=-levels*2; j<=levels*2; j++) {
+ cnt += mkcode(scale*(float)i, scale*(float)j, max, NULL, NULL);
+ }
+ }
+
+ vc = *vcs = _ogg_malloc(2*cnt*sizeof(float));
+
+ /* storage pass */
+ scnt = cnt;
+ cnt = 0;
+ for(i=-levels*2; i<=levels*2; i++) {
+ for(j=-levels*2; j<=levels*2; j++) {
+ if (mkcode(scale*(float)i, scale*(float)j, max, &x, &y)) {
+ printf("%+d %+d %+.4f %+.4f\n", i, j, x, y);
+ vc[cnt++] = x;
+ vc[cnt++] = y;
+ }
+ }
+ }
+ return scnt;
+ }
+
+
+ /* ugly */
+
+ static float MXIDX = 0.0;
+
+ void
+ init_static(static_codebook *c, int dim, int entries, float max) {
+ int bits = 14;
+ float mxidx = (1<<bits)-1;
+ float delta = 2*max/mxidx;
+
+ MXIDX = mxidx;
+
+ c->dim = dim;
+ c->entries = entries;
+ c->lengthlist = _ogg_malloc(sizeof(long) * entries);
+ c->maptype = 2;
+ c->q_min = _float32_pack(-max);
+ c->q_delta = _float32_pack(delta);
+ c->q_quant = bits; /* determine better */
+ c->q_sequencep = 0; /* what's this button do? */
+ c->quantlist = _ogg_malloc(sizeof(long) * dim*entries);
+ c->nearest_tree = NULL;
+ c->thresh_tree = NULL;
+ c->pigeon_tree = NULL;
+ c->allocedp = 0;
+ }
+
+ int qq(double min, double delta, double val) {
+ int r = rint( (val-min)/delta );
+ //printf("r=%d min=%g delta=%g val=%g\n", r, min, delta, val);
+ if (r<0) r = 0;
+ if (r>MXIDX) r = MXIDX;
+ // printf("ERROR: r=%d min=%g delta=%g val=%g\n", r, min,
delta, val);
+ return r;
+ }
+
+
+ void
+ quant_vc(static_codebook *c, float *vc) {
+ int i, j;
+ int entries = c->entries;
+ long *qlist = c->quantlist;
+ float min = _float32_unpack(c->q_min);
+ float delta = _float32_unpack(c->q_delta);
+
+ for(i=0; i<entries; i++) {
+ qlist[2*i] = qq(min, delta, vc[2*i]);
+ qlist[2*i+1] = qq(min, delta, vc[2*i+1]);
+ }
+ }
+
+
+ int main(int argc,char *argv[]){
+ static_codebook c;
+
+ float *vc;
+ long *hist;
+ int entries = -1;
+ int dim = 2;
+ char *name, *fname;
+ long i,j,k;
+ float max;
+ FILE *fp;
+
+ if (argc<3) {
+ fprintf(stderr, "Usage: %s codebook name max\n", argv[0]);
+ exit(2);
+ }
+
+ name = argv[1];
+ fname = malloc(5+strlen(name));
+ strcpy(fname, name);
+ strcat(fname,".vqh");
+
+ if (!(fp = fopen(fname, "w"))) {
+ perror("fopen");
+ exit(2);
+ }
+
+ max = atof(argv[2]);
+ // entries = make_hexlat( max, max, 1.15, &vc);
+ entries = make_hexlat( max, max, 1.3, &vc);
+ // for(i=0; i<entries; i++)
+ // printf("%.4f %4f\n", vc[2*i], vc[2*i+1]);
+
+ fprintf(stderr, "make_hexlat found %d entries inside %f\n",
entries, max);
+
+ init_static(&c, 2, entries, max);
+ quant_vc(&c, vc);
+
+ /* allocate fake histogram */
+ hist = _ogg_malloc(sizeof(long) * entries);
+ for(i=0; i<entries; i++) hist[i]=1;
+
+ build_tree_from_lengths0(entries, hist, c.lengthlist);
+
+ if (getenv("DBOOK")) {
+ codebook b;
+ vorbis_book_init_encode(&b, &c);
+ }
+
+ /* save the book in C header form */
+ write_codebook(fp,name,&c);
+ exit(0);
+ }
diff -N -r -c -b ../libvorbis-1.0beta4/vq/refinehuff.c ./vq/refinehuff.c
*** ../libvorbis-1.0beta4/vq/refinehuff.c Wed Dec 31 19:00:00 1969
--- ./vq/refinehuff.c Tue May 1 18:24:41 2001
***************
*** 0 ****
--- 1,113 ----
+ /********************************************************************
+ * *
+ * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
+ * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
+ * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
+ * *
+ * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
+ * by the XIPHOPHORUS Company http://www.xiph.org/ *
+
+ ********************************************************************
+
+ function: utility main for building codebooks from training sets
+ last mod: $Id: build.c,v 1.20 2001/02/26 03:51:12 xiphmont Exp $
+
+ ********************************************************************/
+
+ #include <stdlib.h>
+ #include <stdio.h>
+ #include <math.h>
+ #include <string.h>
+ #include <errno.h>
+ #include "bookutil.h"
+
+ #include "vqgen.h"
+ #include "vqsplit.h"
+
+ static
+ char *
+ get_long(char *line, long *val) {
+ char *end;
+ *val = strtol(line , &end, 10);
+ while(*end == ',' || isspace(*end) ) end++;
+ return (*end == '\0') ? NULL : end;
+ }
+
+
+ static
+ long *
+ readhist(FILE *fp, int entries) {
+ long *hist = _ogg_calloc(entries, sizeof(long));
+ char *line = NULL;
+
+ while(( line = get_line(fp)) ) {
+ long val;
+ int cnt = 0;
+ while(1) {
+ line = get_long(line, &val);
+ if (cnt == entries) {
+ fprintf(stderr, "Too many entries in histogram\n");
+ exit(2);
+ }
+ hist[cnt++] = val;
+ if (!line) break;
+ }
+ printf("read %d entry histogram\n", cnt);
+ }
+ return hist;
+ }
+
+
+
+
+ int main(int argc,char *argv[]){
+ codebook *c;
+ const static_codebook *cs;
+ long *hist;
+ char *name;
+ FILE *ifp, *ofp, *histp;
+
+ if (argc<3) {
+ fprintf(stderr, "Usage: %s inbook bookname outbook histogram\n",
argv[0]);
+ exit(2);
+ }
+
+ name = argv[1];
+
+ if (!( c = codebook_load(argv[2]) )) {
+ printf("Could not load codebook header `%s'\n", argv[2]);
+ exit(2);
+ }
+
+ cs = c->c;
+
+ printf("read codebook:\n");
+ printf("codebook entries %d\n", cs->entries);
+
+ if (!(ofp = fopen(argv[3], "w"))) {
+ perror("fopen");
+ exit(2);
+ }
+
+ if (!(histp = fopen(argv[4], "r"))) {
+ perror("fopen");
+ exit(2);
+ }
+
+ hist = readhist(histp, cs->entries);
+ printf("read histogram entries\n");
+ {
+ int i, f = 1;
+ for(i=0;i<cs->entries;i++) if (!hist[i]) {
+ if (f) printf("*** Warning histgram has zero entries at
indices:");
+ f = 0;
+ printf(" %d", i);
+ }
+ if (!f) printf("\n");
+ }
+
+ build_tree_from_lengths0(cs->entries, hist, cs->lengthlist);
+ write_codebook(ofp,name,cs);
+ exit(0);
+ }
diff -N -r -c -b ../libvorbis-1.0beta4/vq/vqsplit.h ./vq/vqsplit.h
*** ../libvorbis-1.0beta4/vq/vqsplit.h Wed Dec 31 19:00:00 1969
--- ./vq/vqsplit.h Tue May 1 18:28:13 2001
***************
*** 0 ****
--- 1,37 ----
+ /********************************************************************
+ * *
+ * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
+ * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
+ * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
+ * *
+ * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
+ * by the XIPHOPHORUS Company http://www.xiph.org/ *
+
+ ********************************************************************
+
+ function: build a VQ codebook decision tree
+ last mod: $Id: vqsplit.h,v 1.7 2001/02/26 03:51:13 xiphmont Exp $
+
+ ********************************************************************/
+
+ #ifndef _VQSPL_H_
+ #define _VQSPL_H_
+
+ #include "codebook.h"
+
+ extern void vqsp_book(vqgen *v,codebook *b,long *quantlist);
+ extern int vqenc_entry(codebook *b,float *val);
+ extern int lp_split(float *pointlist,long totalpoints,
+ codebook *b,
+ long *entryindex,long entries,
+ long *pointindex,long points,
+ long *membership,long *reventry,
+ long depth, long *pointsofar);
+
+ #endif
+
+
+
+
+
--- >8 ----
List archives: http://www.xiph.org/archives/
Ogg project homepage: http://www.xiph.org/ogg/
To unsubscribe from this list, send a message to
'vorbis-dev-request@xiph.org'
containing only the word 'unsubscribe' in the body. No subject is
needed.
Unsubscribe messages sent to the list will be ignored/filtered.