tventi: import changes from plan 9 - plan9port - [fork] Plan 9 from user space
(HTM) git clone git://src.adamsgaard.dk/plan9port
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
(DIR) commit f5a8ea6fd8908c6f42670b8546239fdbc7fdbf03
(DIR) parent 7fb06adf54aa6e47974673dcdeb328780927b8e6
(HTM) Author: David du Colombier <0intro@gmail.com>
Date: Thu, 2 Jun 2011 09:33:56 -0400
venti: import changes from plan 9
R=rsc
CC=plan9port.codebot
http://codereview.appspot.com/4523057
Diffstat:
M src/cmd/venti/dump.c | 1 -
M src/cmd/venti/root.c | 10 ++++++++--
M src/cmd/venti/srv/arena.c | 5 ++++-
M src/cmd/venti/srv/arenas.c | 9 +++++++--
M src/cmd/venti/srv/buildindex.c | 21 +++++++++++++++++----
M src/cmd/venti/srv/checkarenas.c | 4 +++-
M src/cmd/venti/srv/config.c | 4 ++--
M src/cmd/venti/srv/conv.c | 10 ++++++++--
M src/cmd/venti/srv/dat.h | 2 ++
M src/cmd/venti/srv/findscore.c | 2 +-
M src/cmd/venti/srv/fmtarenas.c | 2 +-
M src/cmd/venti/srv/fmtindex.c | 4 ++--
M src/cmd/venti/srv/icachewrite.c | 1 -
M src/cmd/venti/srv/index.c | 14 +++++++++-----
M src/cmd/venti/srv/printarena.c | 4 ++--
M src/cmd/venti/srv/printarenapart.c | 2 +-
M src/cmd/venti/srv/rdarena.c | 13 +++++++++----
M src/cmd/venti/srv/syncindex.c | 2 +-
M src/cmd/venti/srv/venti.c | 14 +++++++++++++-
M src/cmd/venti/srv/wrarena.c | 8 ++++----
M src/libavl/avl.c | 72 ++++++++++++++++++++-----------
21 files changed, 141 insertions(+), 63 deletions(-)
---
(DIR) diff --git a/src/cmd/venti/dump.c b/src/cmd/venti/dump.c
t@@ -103,7 +103,6 @@ threadmain(int argc, char *argv[])
fmtinstall('F', vtfcallfmt);
fmtinstall('V', vtscorefmt);
- type = -1;
ARGBEGIN{
case 'h':
host = EARGF(usage());
(DIR) diff --git a/src/cmd/venti/root.c b/src/cmd/venti/root.c
t@@ -4,6 +4,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65536,
+};
+
void
usage(void)
{
t@@ -38,7 +44,7 @@ threadmain(int argc, char *argv[])
if(argc == 0)
usage();
- buf = vtmallocz(8192);
+ buf = vtmallocz(VtMaxLumpSize);
z = vtdial(host);
if(z == nil)
t@@ -52,7 +58,7 @@ threadmain(int argc, char *argv[])
fprint(2, "cannot parse score '%s': %r\n", argv[i]);
continue;
}
- n = vtread(z, score, VtRootType, buf, 8192);
+ n = vtread(z, score, VtRootType, buf, VtMaxLumpSize);
if(n < 0){
fprint(2, "could not read block %V: %r\n", score);
continue;
(DIR) diff --git a/src/cmd/venti/srv/arena.c b/src/cmd/venti/srv/arena.c
t@@ -728,9 +728,12 @@ okarena(Arena *arena)
logerr(ECorrupt, "arena %s uncompressed size inconsistent with used space %lld %d %lld", arena->name, arena->diskstats.uncsize, arena->diskstats.clumps, arena->diskstats.used);
*/
+ /*
+ * this happens; it's harmless.
+ *
if(arena->ctime > arena->wtime)
logerr(ECorrupt, "arena %s creation time after last write time", arena->name);
-
+ */
return ok;
}
(DIR) diff --git a/src/cmd/venti/srv/arenas.c b/src/cmd/venti/srv/arenas.c
t@@ -148,6 +148,7 @@ initarenapart(Part *part)
ap->arenas = MKNZ(Arena*, ap->narenas);
for(i = 0; i < ap->narenas; i++){
+ debugarena = i;
ap->arenas[i] = initarena(part, ap->map[i].start, ap->map[i].stop - ap->map[i].start, ap->blocksize);
if(ap->arenas[i] == nil){
seterr(ECorrupt, "%s: %r", ap->map[i].name);
t@@ -168,8 +169,11 @@ initarenapart(Part *part)
}
}
- for(i = 0; i < ap->narenas; i++)
+ for(i = 0; i < ap->narenas; i++) {
+ debugarena = i;
addarena(ap->arenas[i]);
+ }
+ debugarena = -1;
return ap;
}
t@@ -359,7 +363,8 @@ parseamap(IFile *f, AMapN *amn)
}
n = v;
if(n > MaxAMap){
- seterr(ECorrupt, "illegal number of elements in %s", f->name);
+ seterr(ECorrupt, "illegal number of elements %d in %s",
+ n, f->name);
return -1;
}
am = MKNZ(AMap, n);
(DIR) diff --git a/src/cmd/venti/srv/buildindex.c b/src/cmd/venti/srv/buildindex.c
t@@ -50,18 +50,19 @@ static void arenapartproc(void*);
void
usage(void)
{
- fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n");
+ fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n");
threadexitsall("usage");
}
void
threadmain(int argc, char *argv[])
{
- int fd, i, napart;
+ int fd, i, napart, nfinish, maxdisks;
u32int bcmem, imem;
Config conf;
Part *p;
+ maxdisks = 100000;
ventifmtinstall();
imem = 256*1024*1024;
ARGBEGIN{
t@@ -78,6 +79,9 @@ threadmain(int argc, char *argv[])
case 'M':
imem = unittoull(EARGF(usage()));
break;
+ case 'm': /* temporary - might go away */
+ maxdisks = atoi(EARGF(usage()));
+ break;
default:
usage();
break;
t@@ -146,17 +150,21 @@ threadmain(int argc, char *argv[])
/* start arena procs */
p = nil;
napart = 0;
+ nfinish = 0;
arenadonechan = chancreate(sizeof(void*), 0);
for(i=0; i<ix->narenas; i++){
if(ix->arenas[i]->part != p){
p = ix->arenas[i]->part;
vtproc(arenapartproc, p);
- napart++;
+ if(++napart >= maxdisks){
+ recvp(arenadonechan);
+ nfinish++;
+ }
}
}
/* wait for arena procs to finish */
- for(i=0; i<napart; i++)
+ for(nfinish=0; nfinish<napart; nfinish++)
recvp(arenadonechan);
/* tell index procs to finish */
t@@ -844,6 +852,11 @@ isectproc(void *v)
sysfatal("not enough memory");
nminibuf = nbuf;
}
+ if (nbuf == 0) {
+ fprint(2, "%s: brand-new index, no work to do\n", argv0);
+ threadexitsall(nil);
+ }
+
/* size buffer to use extra memory */
bufsize = MinBufSize;
while(bufsize*2*nbuf <= isectmem && bufsize < MaxBufSize)
(DIR) diff --git a/src/cmd/venti/srv/checkarenas.c b/src/cmd/venti/srv/checkarenas.c
t@@ -128,8 +128,10 @@ threadmain(int argc, char *argv[])
initdcache(8 * MaxDiskBlock);
for(i = 0; i < ap->narenas; i++)
- if(should(ap->arenas[i]->name, argc, argv))
+ if(should(ap->arenas[i]->name, argc, argv)) {
+ debugarena = i;
checkarena(ap->arenas[i], scan, fix);
+ }
if(verbose > 1)
printstats();
(DIR) diff --git a/src/cmd/venti/srv/config.c b/src/cmd/venti/srv/config.c
t@@ -75,7 +75,7 @@ runconfig(char *file, Config *config)
if(readifile(&f, file) < 0)
return -1;
memset(config, 0, sizeof *config);
- config->mem = 0xFFFFFFFFUL;
+ config->mem = Unspecified;
ok = -1;
line = nil;
for(;;){
t@@ -142,7 +142,7 @@ runconfig(char *file, Config *config)
flds[1], file);
break;
}
- if(config->mem != 0xFFFFFFFFUL){
+ if(config->mem != Unspecified){
seterr(EAdmin, "duplicate mem lines in config file %s", file);
break;
}
(DIR) diff --git a/src/cmd/venti/srv/conv.c b/src/cmd/venti/srv/conv.c
t@@ -15,6 +15,8 @@
#define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
#define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
+int debugarena = -1; /* hack to improve error reporting */
+
static struct {
u32int m;
char *s;
t@@ -112,7 +114,9 @@ unpackarena(Arena *arena, u8int *buf)
m = U32GET(p);
if(m != ArenaMagic){
- seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%#lux)", fmtmagic(fbuf, m), ArenaMagic);
+ seterr(ECorrupt, "arena %d has wrong magic number: %s "
+ "expected ArenaMagic (%#lux)", debugarena,
+ fmtmagic(fbuf, m), ArenaMagic);
return -1;
}
p += U32Size;
t@@ -308,7 +312,9 @@ unpackarenahead(ArenaHead *head, u8int *buf)
m = U32GET(p);
if(m != ArenaHeadMagic){
- seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaHeadMagic (%#lux)", fmtmagic(fbuf, m), ArenaHeadMagic);
+ seterr(ECorrupt, "arena %d head has wrong magic number: %s "
+ "expected ArenaHeadMagic (%#lux)", debugarena,
+ fmtmagic(fbuf, m), ArenaHeadMagic);
return -1;
}
(DIR) diff --git a/src/cmd/venti/srv/dat.h b/src/cmd/venti/srv/dat.h
t@@ -54,6 +54,7 @@ enum
MaxIo = 64*1024, /* max size of a single read or write operation */
ICacheBits = 16, /* default bits for indexing icache */
MaxAMap = 31*1024, /* max. allowed arenas in an address mapping; must be < 32*1024 */
+ Unspecified = TWID32,
/*
* return codes from syncarena
t@@ -750,6 +751,7 @@ extern int l1quantum;
extern int ignorebloom;
extern int icacheprefetch;
extern int syncwrites;
+extern int debugarena; /* print in arena error msgs; -1==unknown */
extern Stats *stathist;
extern int nstathist;
(DIR) diff --git a/src/cmd/venti/srv/findscore.c b/src/cmd/venti/srv/findscore.c
t@@ -93,7 +93,7 @@ threadmain(int argc, char *argv[])
file = argv[0];
if(strscore(argv[1], score) < 0)
- sysfatal("bad score %s\n", argv[1]);
+ sysfatal("bad score %s", argv[1]);
part = initpart(file, OREAD|ODIRECT);
if(part == nil)
(DIR) diff --git a/src/cmd/venti/srv/fmtarenas.c b/src/cmd/venti/srv/fmtarenas.c
t@@ -105,7 +105,7 @@ threadmain(int argc, char *argv[])
if(limit >= ap->size || ap->size - limit < MinArenaSize){
limit = ap->size;
if(limit - addr < MinArenaSize)
- sysfatal("bad arena set math: runt arena at %lld,%lld %lld\n", addr, limit, ap->size);
+ sysfatal("bad arena set math: runt arena at %lld,%lld %lld", addr, limit, ap->size);
}
snprint(aname, ANameSize, "%s%d", name, i);
(DIR) diff --git a/src/cmd/venti/srv/fmtindex.c b/src/cmd/venti/srv/fmtindex.c
t@@ -82,11 +82,11 @@ threadmain(int argc, char *argv[])
arenas[n] = ap->arenas[j];
if(n < ix->narenas){
if(arenas[n] != ix->arenas[n])
- sysfatal("mismatched arenas %s and %s at slot %d\n",
+ sysfatal("mismatched arenas %s and %s at slot %d",
arenas[n]->name, ix->arenas[n]->name, n);
amap[n] = ix->amap[n];
if(amap[n].start != addr)
- sysfatal("mis-located arena %s in index %s\n", arenas[n]->name, ix->name);
+ sysfatal("mis-located arena %s in index %s", arenas[n]->name, ix->name);
addr = amap[n].stop;
}else{
amap[n].start = addr;
(DIR) diff --git a/src/cmd/venti/srv/icachewrite.c b/src/cmd/venti/srv/icachewrite.c
t@@ -251,7 +251,6 @@ icachewritecoord(void *v)
as = icachestate();
if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){
/* will not be able to do anything more than last flush - kick disk */
- fprint(2, "icache: nothing to do - kick dcache\n");
trace(TraceProc, "icachewritecoord kick dcache");
kickdcache();
trace(TraceProc, "icachewritecoord kicked dcache");
(DIR) diff --git a/src/cmd/venti/srv/index.c b/src/cmd/venti/srv/index.c
t@@ -328,16 +328,20 @@ newindex(char *name, ISect **sects, int n)
}
if(nb >= ((u64int)1 << 32)){
- seterr(EBug, "index too large");
- return nil;
+ fprint(2, "%s: index is 2^32 blocks or more; ignoring some of it\n",
+ argv0);
+ nb = ((u64int)1 << 32) - 1;
}
div = (((u64int)1 << 32) + nb - 1) / nb;
- ub = (((u64int)1 << 32) - 1) / div + 1;
if(div < 100){
- seterr(EBug, "index divisor too coarse [%lld buckets]", nb);
- return nil;
+ fprint(2, "%s: index divisor %d too coarse; "
+ "index larger than needed, ignoring some of it\n",
+ argv0, div);
+ div = 100;
+ nb = (((u64int)1 << 32) - 1) / (100 - 1);
}
+ ub = (((u64int)1 << 32) - 1) / div + 1;
if(ub > nb){
seterr(EBug, "index initialization math wrong");
return nil;
(DIR) diff --git a/src/cmd/venti/srv/printarena.c b/src/cmd/venti/srv/printarena.c
t@@ -24,7 +24,7 @@ rdarena(Arena *arena, u64int offset)
e = arena->base + arena->size;
if(offset != ~(u64int)0) {
if(offset >= e-a)
- sysfatal("bad offset %llud >= %llud\n",
+ sysfatal("bad offset %llud >= %llud",
offset, e-a);
aa = offset;
} else
t@@ -111,7 +111,7 @@ threadmain(int argc, char *argv[])
head.size, head.clumpmagic);
if(aoffset+head.size > part->size)
- sysfatal("arena is truncated: want %llud bytes have %llud\n",
+ sysfatal("arena is truncated: want %llud bytes have %llud",
head.size, part->size);
partblocksize(part, head.blocksize);
(DIR) diff --git a/src/cmd/venti/srv/printarenapart.c b/src/cmd/venti/srv/printarenapart.c
t@@ -26,7 +26,7 @@ rdarena(Arena *arena, u64int offset)
e = arena->base + arena->size;
if(offset != ~(u64int)0) {
if(offset >= e-a)
- sysfatal("bad offset %llud >= %llud\n",
+ sysfatal("bad offset %llud >= %llud",
offset, e-a);
aa = offset;
} else
(DIR) diff --git a/src/cmd/venti/srv/rdarena.c b/src/cmd/venti/srv/rdarena.c
t@@ -2,7 +2,7 @@
#include "dat.h"
#include "fns.h"
-static int verbose;
+static int verbose, quiet;
void
usage(void)
t@@ -18,8 +18,10 @@ rdarena(Arena *arena)
u64int a, e;
u32int bs;
- fprint(2, "copying %s to standard output\n", arena->name);
- printarena(2, arena);
+ if (!quiet) {
+ fprint(2, "copying %s to standard output\n", arena->name);
+ printarena(2, arena);
+ }
bs = MaxIoSize;
if(bs < arena->blocksize)
t@@ -51,6 +53,9 @@ threadmain(int argc, char *argv[])
statsinit();
ARGBEGIN{
+ case 'q':
+ quiet++;
+ break;
case 'v':
verbose++;
break;
t@@ -87,5 +92,5 @@ threadmain(int argc, char *argv[])
}
}
- sysfatal("couldn't find arena %s\n", aname);
+ sysfatal("couldn't find arena %s", aname);
}
(DIR) diff --git a/src/cmd/venti/srv/syncindex.c b/src/cmd/venti/srv/syncindex.c
t@@ -56,7 +56,7 @@ threadmain(int argc, char *argv[])
if(verbose)
printindex(2, mainindex);
if(syncindex(mainindex) < 0)
- sysfatal("failed to sync index=%s: %r\n", mainindex->name);
+ sysfatal("failed to sync index=%s: %r", mainindex->name);
flushicache();
flushdcache();
(DIR) diff --git a/src/cmd/venti/srv/venti.c b/src/cmd/venti/srv/venti.c
t@@ -109,6 +109,9 @@ threadmain(int argc, char *argv[])
fprint(2, "conf...");
if(initventi(configfile, &config) < 0)
sysfatal("can't init server: %r");
+ /*
+ * load bloom filter
+ */
if(mainindex->bloom && loadbloom(mainindex->bloom) < 0)
sysfatal("can't load bloom filter: %r");
t@@ -139,15 +142,22 @@ threadmain(int argc, char *argv[])
if(mem == 0xffffffffUL)
mem = 1 * 1024 * 1024;
+
+ /*
+ * lump cache
+ */
if(0) fprint(2, "initialize %d bytes of lump cache for %d lumps\n",
mem, mem / (8 * 1024));
initlumpcache(mem, mem / (8 * 1024));
+ /*
+ * index cache
+ */
initicache(icmem);
initicachewrite();
/*
- * need a block for every arena and every process
+ * block cache: need a block for every arena and every process
*/
minbcmem = maxblocksize *
(mainindex->narenas + mainindex->nsects*4 + 16);
t@@ -186,6 +196,8 @@ threadmain(int argc, char *argv[])
ventiserver(nil);
else
vtproc(ventiserver, nil);
+
+ threadexits(nil);
}
static void
(DIR) diff --git a/src/cmd/venti/srv/wrarena.c b/src/cmd/venti/srv/wrarena.c
t@@ -75,7 +75,7 @@ rdarena(Arena *arena, u64int offset)
e = arena->base + arena->size;
if(offset != ~(u64int)0) {
if(offset >= e - a)
- sysfatal("bad offset %#llx >= %#llx\n", offset, e - a);
+ sysfatal("bad offset %#llx >= %#llx", offset, e - a);
aa = offset;
} else
aa = 0;
t@@ -87,8 +87,8 @@ rdarena(Arena *arena, u64int offset)
break;
if(a < aa || ci.type == VtCorruptType){
if(ci.type == VtCorruptType)
- fprint(2, "corrupt at %#llx: +%d\n",
- a, ClumpSize+ci.size);
+ fprint(2, "%s: corrupt clump read at %#llx: +%d\n",
+ argv0, a, ClumpSize+ci.size);
continue;
}
lump = loadclump(arena, a, 0, &cl, score, 0);
t@@ -187,7 +187,7 @@ threadmain(int argc, char *argv[])
sysfatal("corrupted arena header: %r");
if(aoffset+head.size > part->size)
- sysfatal("arena is truncated: want %llud bytes have %llud\n",
+ sysfatal("arena is truncated: want %llud bytes have %llud",
head.size, part->size);
partblocksize(part, head.blocksize);
(DIR) diff --git a/src/libavl/avl.c b/src/libavl/avl.c
t@@ -97,6 +97,16 @@ balance(Avl **tp, Avl *p)
}
static int
+canoncmp(int cmp)
+{
+ if(cmp < 0)
+ return -1;
+ else if(cmp > 0)
+ return 1;
+ return 0;
+}
+
+static int
_insertavl(Avl **tp, Avl *p, Avl *r, int (*cmp)(Avl*,Avl*), Avl **rfree)
{
int i, ob;
t@@ -110,7 +120,7 @@ _insertavl(Avl **tp, Avl *p, Avl *r, int (*cmp)(Avl*,Avl*), Avl **rfree)
return 1;
}
ob = (*tp)->bal;
- if((i = cmp(r, *tp)) != 0){
+ if((i = canoncmp(cmp(r, *tp))) != 0){
(*tp)->bal += i * _insertavl(&(*tp)->n[(i+1)/2], *tp, r, cmp,
rfree);
balance(tp, p);
t@@ -129,23 +139,6 @@ _insertavl(Avl **tp, Avl *p, Avl *r, int (*cmp)(Avl*,Avl*), Avl **rfree)
return 0;
}
-static Avl*
-_lookupavl(Avl *t, Avl *r, int (*cmp)(Avl*,Avl*))
-{
- int i;
- Avl *p;
-
- p = nil;
- while(t != nil){
- assert(t->p == p);
- if((i = cmp(r, t)) == 0)
- return t;
- p = t;
- t = t->n[(i+1)/2];
- }
- return nil;
-}
-
static int
successor(Avl **tp, Avl *p, Avl **r)
{
t@@ -175,7 +168,7 @@ _deleteavl(Avl **tp, Avl *p, Avl *rx, int(*cmp)(Avl*,Avl*), Avl **del,
return 0;
ob = (*tp)->bal;
- if((i=cmp(rx, *tp)) != 0){
+ if((i=canoncmp(cmp(rx, *tp))) != 0){
(*tp)->bal += i * _deleteavl(&(*tp)->n[(i+1)/2], *tp, rx, cmp,
del, predel, arg);
balance(tp, p);
t@@ -259,12 +252,6 @@ insertavl(Avltree *t, Avl *new, Avl **oldp)
_insertavl(&t->root, nil, new, t->cmp, oldp);
}
-Avl*
-lookupavl(Avltree *t, Avl *key)
-{
- return _lookupavl(t->root, key, t->cmp);
-}
-
static Avl*
findpredecessor(Avl *a)
{
t@@ -303,6 +290,41 @@ findsuccessor(Avl *a)
}
}
+static Avl*
+_lookupavl(Avl *t, Avl *r, int (*cmp)(Avl*,Avl*), int neighbor)
+{
+ int i;
+ Avl *p;
+
+ p = nil;
+ if(t == nil)
+ return nil;
+ do{
+ assert(t->p == p);
+ if((i = canoncmp(cmp(r, t))) == 0)
+ return t;
+ p = t;
+ t = t->n[(i+1)/2];
+ }while(t);
+ if(neighbor == 0)
+ return nil;
+ if(neighbor < 0)
+ return i > 0 ? p : findpredecessor(p);
+ return i < 0 ? p : findsuccessor(p);
+}
+
+Avl*
+searchavl(Avltree *t, Avl *key, int neighbor)
+{
+ return _lookupavl(t->root, key, t->cmp, neighbor);
+}
+
+Avl*
+lookupavl(Avltree *t, Avl *key)
+{
+ return _lookupavl(t->root, key, t->cmp, 0);
+}
+
static void
walkdel(Avl *a, void *v)
{