tventi: 32-bit extensions to data structures - plan9port - [fork] Plan 9 from user space
(HTM) git clone git://src.adamsgaard.dk/plan9port
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
(DIR) commit 75d048884cfcb7cc4404b384da50923e22224365
(DIR) parent 33b446b8bbfea80552d462296d27ad4114fbd3fb
(HTM) Author: Russ Cox <rsc@swtch.com>
Date: Mon, 25 May 2009 02:11:27 -0700
venti: 32-bit extensions to data structures
Diffstat:
M include/venti.h | 24 +++++++++++++-----------
M src/cmd/vac/file.c | 34 +++++++++++++++++++++----------
M src/cmd/vac/pack.c | 2 +-
M src/cmd/vac/unvac.c | 2 +-
M src/cmd/vac/vac.c | 14 +++++++-------
M src/cmd/vac/vac.h | 6 +++---
M src/cmd/vac/vacfs.c | 37 ++++++++++++++++++++++++++-----
M src/cmd/venti/copy.c | 6 ++++++
M src/cmd/venti/dump.c | 6 ++++++
M src/cmd/venti/read.c | 6 ++++++
M src/cmd/venti/root.c | 4 ++--
M src/cmd/venti/srv/dat.h | 6 ++++++
M src/cmd/venti/write.c | 6 ++++++
M src/libdiskfs/venti.c | 8 ++++----
M src/libventi/cache.c | 110 ++++++++++++++++++++++---------
M src/libventi/entry.c | 56 ++++++++++++++++++++++++++-----
M src/libventi/file.c | 53 +++++++++++++++----------------
M src/libventi/root.c | 21 +++++++++++++++++----
18 files changed, 283 insertions(+), 118 deletions(-)
---
(DIR) diff --git a/include/venti.h b/include/venti.h
t@@ -86,7 +86,6 @@ enum
{
VtScoreSize = 20,
VtMaxStringSize = 1024,
- VtMaxLumpSize = 56*1024,
VtPointerDepth = 7
};
#define VtMaxFileSize ((1ULL<<48)-1)
t@@ -137,7 +136,8 @@ enum
_VtEntryDir = 1<<1, /* a directory */
_VtEntryDepthShift = 2, /* shift for pointer depth */
_VtEntryDepthMask = 7<<2, /* mask for pointer depth */
- VtEntryLocal = 1<<5 /* for local storage only */
+ VtEntryLocal = 1<<5, /* for local storage only */
+ _VtEntryBig = 1<<6,
};
enum
{
t@@ -146,8 +146,8 @@ enum
struct VtEntry
{
ulong gen; /* generation number */
- ushort psize; /* pointer block size */
- ushort dsize; /* data block size */
+ ulong psize; /* pointer block size */
+ ulong dsize; /* data block size */
uchar type;
uchar flags;
uvlong size;
t@@ -162,14 +162,15 @@ struct VtRoot
char name[128];
char type[128];
uchar score[VtScoreSize]; /* to a Dir block */
- ushort blocksize; /* maximum block size */
+ ulong blocksize; /* maximum block size */
uchar prev[VtScoreSize]; /* last root block */
};
enum
{
VtRootSize = 300,
- VtRootVersion = 2
+ VtRootVersion = 2,
+ _VtRootVersionBig = 1<<15,
};
void vtrootpack(VtRoot*, uchar*);
t@@ -394,7 +395,8 @@ struct VtBlock
uchar *data;
uchar score[VtScoreSize];
- uchar type; /* BtXXX */
+ uchar type; /* VtXXX */
+ ulong size;
/* internal to cache */
int nlock;
t@@ -412,14 +414,13 @@ struct VtBlock
u32int vtglobaltolocal(uchar[VtScoreSize]);
void vtlocaltoglobal(u32int, uchar[VtScoreSize]);
-VtCache *vtcachealloc(VtConn*, int blocksize, ulong nblocks);
+VtCache *vtcachealloc(VtConn*, ulong maxmem);
void vtcachefree(VtCache*);
VtBlock *vtcachelocal(VtCache*, u32int addr, int type);
-VtBlock *vtcacheglobal(VtCache*, uchar[VtScoreSize], int type);
-VtBlock *vtcacheallocblock(VtCache*, int type);
+VtBlock *vtcacheglobal(VtCache*, uchar[VtScoreSize], int type, ulong size);
+VtBlock *vtcacheallocblock(VtCache*, int type, ulong size);
void vtcachesetwrite(VtCache*, int(*)(VtConn*,uchar[VtScoreSize],uint,uchar*,int));
void vtblockput(VtBlock*);
-u32int vtcacheblocksize(VtCache*);
int vtblockwrite(VtBlock*);
VtBlock *vtblockcopy(VtBlock*);
void vtblockduplock(VtBlock*);
t@@ -438,6 +439,7 @@ struct VtFile
int local;
VtBlock *b; /* block containing this file */
uchar score[VtScoreSize]; /* score of block containing this file */
+ int bsize; /* size of block */
/* immutable */
VtCache *c;
(DIR) diff --git a/src/cmd/vac/file.c b/src/cmd/vac/file.c
t@@ -1730,7 +1730,7 @@ Err1:
static char EBadVacFormat[] = "bad format for vac file";
static VacFs *
-vacfsalloc(VtConn *z, int bsize, int ncache, int mode)
+vacfsalloc(VtConn *z, int bsize, ulong cachemem, int mode)
{
VacFs *fs;
t@@ -1738,7 +1738,7 @@ vacfsalloc(VtConn *z, int bsize, int ncache, int mode)
fs->z = z;
fs->bsize = bsize;
fs->mode = mode;
- fs->cache = vtcachealloc(z, bsize, ncache);
+ fs->cache = vtcachealloc(z, cachemem);
return fs;
}
t@@ -1767,7 +1767,7 @@ readscore(int fd, uchar score[VtScoreSize])
}
VacFs*
-vacfsopen(VtConn *z, char *file, int mode, int ncache)
+vacfsopen(VtConn *z, char *file, int mode, ulong cachemem)
{
int fd;
uchar score[VtScoreSize];
t@@ -1788,11 +1788,11 @@ vacfsopen(VtConn *z, char *file, int mode, int ncache)
}
close(fd);
}
- return vacfsopenscore(z, score, mode, ncache);
+ return vacfsopenscore(z, score, mode, cachemem);
}
VacFs*
-vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache)
+vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem)
{
VacFs *fs;
int n;
t@@ -1818,13 +1818,19 @@ vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache)
return nil;
}
- fs = vacfsalloc(z, rt.blocksize, ncache, mode);
+ fs = vacfsalloc(z, rt.blocksize, cachemem, mode);
memmove(fs->score, score, VtScoreSize);
fs->mode = mode;
memmove(e.score, rt.score, VtScoreSize);
e.gen = 0;
+
+ // Don't waste cache memory on directories
+ // when rt.blocksize is large.
e.psize = (rt.blocksize/VtEntrySize)*VtEntrySize;
+ if(e.psize > 60000)
+ e.psize = (60000/VtEntrySize)*VtEntrySize;
+
e.dsize = rt.blocksize;
e.type = VtDirType;
e.flags = VtEntryActive;
t@@ -1925,7 +1931,7 @@ vacfsclose(VacFs *fs)
* Create a fresh vac fs.
*/
VacFs *
-vacfscreate(VtConn *z, int bsize, int ncache)
+vacfscreate(VtConn *z, int bsize, ulong cachemem)
{
VacFs *fs;
VtFile *f;
t@@ -1937,18 +1943,24 @@ vacfscreate(VtConn *z, int bsize, int ncache)
MetaEntry me;
int psize;
- if((fs = vacfsalloc(z, bsize, ncache, VtORDWR)) == nil)
+ if((fs = vacfsalloc(z, bsize, cachemem, VtORDWR)) == nil)
return nil;
-
+
/*
* Fake up an empty vac fs.
*/
psize = bsize/VtEntrySize*VtEntrySize;
+ if(psize > 60000)
+ psize = 60000/VtEntrySize*VtEntrySize;
+fprint(2, "create bsize %d psize %d\n", bsize, psize);
+
f = vtfilecreateroot(fs->cache, psize, bsize, VtDirType);
+ if(f == nil)
+ sysfatal("vtfilecreateroot: %r");
vtfilelock(f, VtORDWR);
-
+
/* Write metablock containing root directory VacDir. */
- b = vtcacheallocblock(fs->cache, VtDataType);
+ b = vtcacheallocblock(fs->cache, VtDataType, bsize);
mbinit(&mb, b->data, bsize, bsize/BytesPerEntry);
memset(&vd, 0, sizeof vd);
vd.elem = "/";
(DIR) diff --git a/src/cmd/vac/pack.c b/src/cmd/vac/pack.c
t@@ -77,7 +77,7 @@ mbunpack(MetaBlock *mb, uchar *p, int n)
magic = U32GET(p);
if(magic != MetaMagic && magic != MetaMagic+1) {
- werrstr("bad meta block magic");
+ werrstr("bad meta block magic %#08ux", magic);
return -1;
}
mb->size = U16GET(p+4);
(DIR) diff --git a/src/cmd/vac/unvac.c b/src/cmd/vac/unvac.c
t@@ -94,7 +94,7 @@ threadmain(int argc, char *argv[])
if(vtconnect(conn) < 0)
sysfatal("vtconnect: %r");
- fs = vacfsopen(conn, argv[0], VtOREAD, 128);
+ fs = vacfsopen(conn, argv[0], VtOREAD, 4<<20);
if(fs == nil)
sysfatal("vacfsopen: %r");
(DIR) diff --git a/src/cmd/vac/vac.c b/src/cmd/vac/vac.c
t@@ -100,8 +100,6 @@ threadmain(int argc, char **argv)
u = unittoull(EARGF(usage()));
if(u < 512)
u = 512;
- if(u > VtMaxLumpSize)
- u = VtMaxLumpSize;
blocksize = u;
break;
case 'd':
t@@ -170,10 +168,10 @@ threadmain(int argc, char **argv)
if((outfd = create(archivefile, OWRITE, 0666)) < 0)
sysfatal("create %s: %r", archivefile);
atexit(removevacfile); // because it is new
- if((fs = vacfscreate(z, blocksize, 512)) == nil)
+ if((fs = vacfscreate(z, blocksize, 4<<20)) == nil)
sysfatal("vacfscreate: %r");
}else{
- if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil)
+ if((fs = vacfsopen(z, archivefile, VtORDWR, 4<<20)) == nil)
sysfatal("vacfsopen %s: %r", archivefile);
if((fdiff = recentarchive(fs, oldpath)) != nil){
if(verbose)
t@@ -213,7 +211,7 @@ threadmain(int argc, char **argv)
else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
sysfatal("create %s: %r", vacfile);
atexit(removevacfile);
- if((fs = vacfscreate(z, blocksize, 512)) == nil)
+ if((fs = vacfscreate(z, blocksize, 4<<20)) == nil)
sysfatal("vacfscreate: %r");
f = vacfsgetroot(fs);
t@@ -450,7 +448,7 @@ void
vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
{
char *elem, *s;
- static char buf[65536];
+ static char *buf;
int fd, i, n, bsize;
vlong off;
Dir *dk; // kids
t@@ -541,6 +539,8 @@ vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
}else{
off = 0;
bsize = fs->bsize;
+ if(buf == nil)
+ buf = vtmallocz(bsize);
if(fdiff){
/*
* Copy fdiff's contents into f by moving the score.
t@@ -708,7 +708,7 @@ vacmerge(VacFile *fp, char *name)
if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
return -1;
- if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil)
+ if((mfs = vacfsopen(z, name, VtOREAD, 4<<20)) == nil)
return -1;
if(verbose)
fprint(2, "merging %s\n", name);
(DIR) diff --git a/src/cmd/vac/vac.h b/src/cmd/vac/vac.h
t@@ -97,9 +97,9 @@ struct VacFs
VtCache *cache;
};
-VacFs *vacfsopen(VtConn *z, char *file, int mode, int ncache);
-VacFs *vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache);
-VacFs *vacfscreate(VtConn *z, int bsize, int ncache);
+VacFs *vacfsopen(VtConn *z, char *file, int mode, ulong cachemem);
+VacFs *vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem);
+VacFs *vacfscreate(VtConn *z, int bsize, ulong cachemem);
void vacfsclose(VacFs *fs);
int vacfssync(VacFs *fs);
int vacfssnapshot(VacFs *fs, char *src, char *dst);
(DIR) diff --git a/src/cmd/vac/vacfs.c b/src/cmd/vac/vacfs.c
t@@ -118,6 +118,31 @@ notifyf(void *a, char *s)
noted(NDFLT);
}
+#define TWID64 ~(u64int)0
+static u64int
+unittoull(char *s)
+{
+ char *es;
+ u64int n;
+
+ if(s == nil)
+ return TWID64;
+ n = strtoul(s, &es, 0);
+ if(*es == 'k' || *es == 'K'){
+ n *= 1024;
+ es++;
+ }else if(*es == 'm' || *es == 'M'){
+ n *= 1024*1024;
+ es++;
+ }else if(*es == 'g' || *es == 'G'){
+ n *= 1024*1024*1024;
+ es++;
+ }
+ if(*es != '\0')
+ return TWID64;
+ return n;
+}
+
void
threadmain(int argc, char *argv[])
{
t@@ -125,10 +150,10 @@ threadmain(int argc, char *argv[])
int p[2], fd;
int stdio;
char *host = nil;
- long ncache;
+ ulong mem;
+ mem = 16<<20;
stdio = 0;
- ncache = 256;
fmtinstall('H', encodefmt);
fmtinstall('V', vtscorefmt);
fmtinstall('F', vtfcallfmt);
t@@ -140,9 +165,6 @@ threadmain(int argc, char *argv[])
fmtinstall('F', fcallfmt);
dflag = 1;
break;
- case 'c':
- ncache = atoi(EARGF(usage()));
- break;
case 'i':
defmnt = nil;
stdio = 1;
t@@ -158,6 +180,9 @@ threadmain(int argc, char *argv[])
case 's':
defsrv = "vacfs";
break;
+ case 'M':
+ mem = unittoull(EARGF(usage()));
+ break;
case 'm':
defmnt = EARGF(usage());
break;
t@@ -206,7 +231,7 @@ threadmain(int argc, char *argv[])
if(vtconnect(conn) < 0)
sysfatal("vtconnect: %r");
- fs = vacfsopen(conn, argv[0], VtOREAD, ncache);
+ fs = vacfsopen(conn, argv[0], VtOREAD, mem);
if(fs == nil)
sysfatal("vacfsopen: %r");
(DIR) diff --git a/src/cmd/venti/copy.c b/src/cmd/venti/copy.c
t@@ -4,6 +4,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65536,
+};
+
int changes;
int rewrite;
int ignoreerrors;
(DIR) diff --git a/src/cmd/venti/dump.c b/src/cmd/venti/dump.c
t@@ -5,6 +5,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65536,
+};
+
VtConn *z;
char *host;
(DIR) diff --git a/src/cmd/venti/read.c b/src/cmd/venti/read.c
t@@ -4,6 +4,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65536,
+};
+
void
usage(void)
{
(DIR) diff --git a/src/cmd/venti/root.c b/src/cmd/venti/root.c
t@@ -38,7 +38,7 @@ threadmain(int argc, char *argv[])
if(argc == 0)
usage();
- buf = vtmallocz(VtMaxLumpSize);
+ buf = vtmallocz(8192);
z = vtdial(host);
if(z == nil)
t@@ -52,7 +52,7 @@ threadmain(int argc, char *argv[])
fprint(2, "cannot parse score '%s': %r\n", argv[i]);
continue;
}
- n = vtread(z, score, VtRootType, buf, VtMaxLumpSize);
+ n = vtread(z, score, VtRootType, buf, 8192);
if(n < 0){
fprint(2, "could not read block %V: %r\n", score);
continue;
(DIR) diff --git a/src/cmd/venti/srv/dat.h b/src/cmd/venti/srv/dat.h
t@@ -37,6 +37,12 @@ typedef struct Bloom Bloom;
enum
{
+ /*
+ * formerly fundamental constant,
+ * now a server-imposed limitation.
+ */
+ VtMaxLumpSize = 56*1024,
+
ABlockLog = 9, /* log2(512), the quantum for reading arenas */
ANameSize = 64,
MaxDiskBlock = 64*1024, /* max. allowed size for a disk block */
(DIR) diff --git a/src/cmd/venti/write.c b/src/cmd/venti/write.c
t@@ -4,6 +4,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65536,
+};
+
void
usage(void)
{
(DIR) diff --git a/src/libdiskfs/venti.c b/src/libdiskfs/venti.c
t@@ -96,13 +96,13 @@ _vtfileblock(VtCache *c, VtEntry *e, u32int bn)
}
/*fprint(2, "vtread %V\n", e->score); */
- b = vtcacheglobal(c, e->score, e->type);
+ b = vtcacheglobal(c, e->score, e->type, d == 0 ? e->dsize : e->psize);
for(i=d-1; i>=0 && b; i--){
t = VtDataType+i;
/*fprint(2, "vtread %V\n", b->data+index[i]*VtScoreSize); */
memmove(score, b->data+index[i]*VtScoreSize, VtScoreSize);
vtblockput(b);
- b = vtcacheglobal(c, score, t);
+ b = vtcacheglobal(c, score, t, i == 0 ? e->dsize : e->psize);
}
return b;
}
t@@ -122,7 +122,7 @@ diskopenventi(VtCache *c, uchar score[VtScoreSize])
VtRoot root;
VtBlock *b;
- if((b = vtcacheglobal(c, score, VtRootType)) == nil)
+ if((b = vtcacheglobal(c, score, VtRootType, VtRootSize)) == nil)
goto Err;
if(vtrootunpack(&root, b->data) < 0)
goto Err;
t@@ -132,7 +132,7 @@ diskopenventi(VtCache *c, uchar score[VtScoreSize])
}
vtblockput(b);
- if((b = vtcacheglobal(c, root.score, VtDirType)) == nil)
+ if((b = vtcacheglobal(c, root.score, VtDirType, VtEntrySize)) == nil)
goto Err;
if(vtentryunpack(&e, b->data, 0) < 0)
goto Err;
(DIR) diff --git a/src/libventi/cache.c b/src/libventi/cache.c
t@@ -32,7 +32,6 @@ struct VtCache
{
QLock lk;
VtConn *z;
- u32int blocksize;
u32int now; /* ticks for usage time stamps */
VtBlock **hash; /* hash table for finding addresses */
int nhash;
t@@ -40,41 +39,45 @@ struct VtCache
int nheap;
VtBlock *block; /* all allocated blocks */
int nblock;
- uchar *mem; /* memory for all blocks and data */
int (*write)(VtConn*, uchar[VtScoreSize], uint, uchar*, int);
+ VtBlock *dead; /* blocks we don't have memory for */
+ ulong mem;
+ ulong maxmem;
};
static void cachecheck(VtCache*);
VtCache*
-vtcachealloc(VtConn *z, int blocksize, ulong nblock)
+vtcachealloc(VtConn *z, ulong maxmem)
{
- uchar *p;
VtCache *c;
int i;
+ int nblock;
VtBlock *b;
+ ulong maxmem0;
+ maxmem0 = maxmem;
c = vtmallocz(sizeof(VtCache));
-
+ nblock = maxmem/100/(sizeof(VtBlock)+2*sizeof(VtBlock*));
c->z = z;
- c->blocksize = (blocksize + 127) & ~127;
c->nblock = nblock;
c->nhash = nblock;
c->hash = vtmallocz(nblock*sizeof(VtBlock*));
c->heap = vtmallocz(nblock*sizeof(VtBlock*));
c->block = vtmallocz(nblock*sizeof(VtBlock));
- c->mem = vtmallocz(nblock*c->blocksize);
c->write = vtwrite;
+ maxmem -= nblock*(sizeof(VtBlock) + 2*sizeof(VtBlock*));
+ maxmem -= sizeof(VtCache);
+ if((long)maxmem < 0)
+ sysfatal("cache size far too small: %lud", maxmem0);
+ c->mem = maxmem;
- p = c->mem;
for(i=0; i<nblock; i++){
b = &c->block[i];
b->addr = NilBlock;
b->c = c;
- b->data = p;
b->heap = i;
c->heap[i] = b;
- p += c->blocksize;
}
c->nheap = nblock;
cachecheck(c);
t@@ -102,13 +105,14 @@ vtcachefree(VtCache *c)
qlock(&c->lk);
cachecheck(c);
- for(i=0; i<c->nblock; i++)
+ for(i=0; i<c->nblock; i++) {
assert(c->block[i].ref == 0);
+ vtfree(c->block[i].data);
+ }
vtfree(c->hash);
vtfree(c->heap);
vtfree(c->block);
- vtfree(c->mem);
vtfree(c);
}
t@@ -128,11 +132,10 @@ vtcachedump(VtCache *c)
static void
cachecheck(VtCache *c)
{
- u32int size, now;
+ u32int now;
int i, k, refed;
VtBlock *b;
- size = c->blocksize;
now = c->now;
for(i = 0; i < c->nheap; i++){
t@@ -151,8 +154,6 @@ cachecheck(VtCache *c)
refed = 0;
for(i = 0; i < c->nblock; i++){
b = &c->block[i];
- if(b->data != &c->mem[i * size])
- sysfatal("mis-blocked at %d", i);
if(b->ref && b->heap == BadHeap)
refed++;
else if(b->addr != NilBlock)
t@@ -299,6 +300,57 @@ if(0)fprint(2, "droping %x:%V\n", b->addr, b->score);
}
/*
+ * evict blocks until there is enough memory for size bytes.
+ */
+static VtBlock*
+vtcacheevict(VtCache *c, ulong size)
+{
+ VtBlock *b;
+
+ /*
+ * If we were out of memory and put some blocks
+ * to the side but now we have memory, grab one.
+ */
+ if(c->mem >= size && c->dead) {
+ b = c->dead;
+ c->dead = b->next;
+ b->next = nil;
+ goto alloc;
+ }
+
+ /*
+ * Otherwise, evict until we have memory.
+ */
+ for(;;) {
+ b = vtcachebumpblock(c);
+ if(c->mem+b->size >= size)
+ break;
+ /*
+ * chain b onto dead list
+ */
+ free(b->data);
+ b->data = nil;
+ c->mem += b->size;
+ b->size = 0;
+ b->next = c->dead;
+ c->dead = b;
+ }
+
+ /*
+ * Allocate memory for block.
+ */
+alloc:
+ if(size > b->size || size <= b->size/2) {
+ free(b->data);
+ c->mem += b->size;
+ c->mem -= size;
+ b->size = size;
+ b->data = vtmalloc(size);
+ }
+ return b;
+}
+
+/*
* fetch a local block from the memory cache.
* if it's not there, load it, bumping some other Block.
* if we're out of free blocks, we're screwed.
t@@ -332,16 +384,16 @@ vtcachelocal(VtCache *c, u32int addr, int type)
}
VtBlock*
-vtcacheallocblock(VtCache *c, int type)
+vtcacheallocblock(VtCache *c, int type, ulong size)
{
VtBlock *b;
qlock(&c->lk);
- b = vtcachebumpblock(c);
+ b = vtcacheevict(c, size);
b->iostate = BioLocal;
b->type = type;
b->addr = (b - c->block)+1;
- vtzeroextend(type, b->data, 0, c->blocksize);
+ vtzeroextend(type, b->data, 0, size);
vtlocaltoglobal(b->addr, b->score);
qunlock(&c->lk);
t@@ -356,7 +408,7 @@ vtcacheallocblock(VtCache *c, int type)
* if it's not there, load it, bumping some other block.
*/
VtBlock*
-vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
+vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type, ulong size)
{
VtBlock *b;
ulong h;
t@@ -409,7 +461,7 @@ vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
/*
* not found
*/
- b = vtcachebumpblock(c);
+ b = vtcacheevict(c, size);
b->addr = NilBlock;
b->type = type;
memmove(b->score, score, VtScoreSize);
t@@ -435,7 +487,7 @@ vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
qunlock(&c->lk);
vtcachenread++;
- n = vtread(c->z, score, type, b->data, c->blocksize);
+ n = vtread(c->z, score, type, b->data, size);
if(n < 0){
if(chattyventi)
fprint(2, "read %V: %r\n", score);
t@@ -445,7 +497,7 @@ vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
vtblockput(b);
return nil;
}
- vtzeroextend(type, b->data, n, c->blocksize);
+ vtzeroextend(type, b->data, n, size);
b->iostate = BioVenti;
b->nlock = 1;
if(vttracelevel)
t@@ -534,7 +586,7 @@ vtblockwrite(VtBlock *b)
}
c = b->c;
- n = vtzerotruncate(b->type, b->data, c->blocksize);
+ n = vtzerotruncate(b->type, b->data, b->size);
vtcachenwrite++;
if(c->write(c->z, score, b->type, b->data, n) < 0)
return -1;
t@@ -554,24 +606,18 @@ vtblockwrite(VtBlock *b)
return 0;
}
-uint
-vtcacheblocksize(VtCache *c)
-{
- return c->blocksize;
-}
-
VtBlock*
vtblockcopy(VtBlock *b)
{
VtBlock *bb;
vtcachencopy++;
- bb = vtcacheallocblock(b->c, b->type);
+ bb = vtcacheallocblock(b->c, b->type, b->size);
if(bb == nil){
vtblockput(b);
return nil;
}
- memmove(bb->data, b->data, b->c->blocksize);
+ memmove(bb->data, b->data, b->size);
vtblockput(b);
bb->pc = getcallerpc(&b);
return bb;
(DIR) diff --git a/src/libventi/entry.c b/src/libventi/entry.c
t@@ -6,13 +6,37 @@
static int
checksize(int n)
{
- if(n < 256 || n > VtMaxLumpSize) {
+ if(n < 256) {
werrstr("bad block size %#ux", n);
return -1;
}
return 0;
}
+// _VtEntryBig integer format is floating-point:
+// (n>>5) << (n&31).
+// Convert this number; must be exact or return -1.
+int
+vttobig(ulong n)
+{
+ int shift;
+ ulong n0;
+
+ n0 = n;
+ shift = 0;
+ while(n >= (1<<(16 - 5))) {
+ if(n & 1)
+ return -1;
+ shift++;
+ n >>= 1;
+ }
+
+ n = (n<<5) | shift;
+ if(((n>>5)<<(n&31)) != n0)
+ sysfatal("vttobig %#lux => %#lux failed", n0, n);
+ return n;
+}
+
void
vtentrypack(VtEntry *e, uchar *p, int index)
{
t@@ -20,21 +44,31 @@ vtentrypack(VtEntry *e, uchar *p, int index)
int flags;
uchar *op;
int depth;
+ int psize, dsize;
p += index * VtEntrySize;
op = p;
- U32PUT(p, e->gen);
- p += 4;
- U16PUT(p, e->psize);
- p += 2;
- U16PUT(p, e->dsize);
- p += 2;
depth = e->type&VtTypeDepthMask;
flags = (e->flags&~(_VtEntryDir|_VtEntryDepthMask));
flags |= depth << _VtEntryDepthShift;
if(e->type - depth == VtDirType)
flags |= _VtEntryDir;
+ U32PUT(p, e->gen);
+ p += 4;
+ psize = e->psize;
+ dsize = e->dsize;
+ if(psize >= (1<<16) || dsize >= (1<<16)) {
+ flags |= _VtEntryBig;
+ psize = vttobig(psize);
+ dsize = vttobig(dsize);
+ if(psize < 0 || dsize < 0)
+ sysfatal("invalid entry psize/dsize: %d/%d", e->psize, e->dsize);
+ }
+ U16PUT(p, psize);
+ p += 2;
+ U16PUT(p, dsize);
+ p += 2;
U8PUT(p, flags);
p++;
memset(p, 0, 5);
t@@ -62,10 +96,14 @@ vtentryunpack(VtEntry *e, uchar *p, int index)
e->dsize = U16GET(p);
p += 2;
e->flags = U8GET(p);
+ p++;
+ if(e->flags & _VtEntryBig) {
+ e->psize = (e->psize>>5)<<(e->psize & 31);
+ e->dsize = (e->dsize>>5)<<(e->dsize & 31);
+ }
e->type = (e->flags&_VtEntryDir) ? VtDirType : VtDataType;
e->type += (e->flags & _VtEntryDepthMask) >> _VtEntryDepthShift;
- e->flags &= ~(_VtEntryDir|_VtEntryDepthMask);
- p++;
+ e->flags &= ~(_VtEntryDir|_VtEntryDepthMask|_VtEntryBig);
p += 5;
e->size = U48GET(p);
p += 6;
(DIR) diff --git a/src/libventi/file.c b/src/libventi/file.c
t@@ -36,7 +36,6 @@ static VtFile *
vtfilealloc(VtCache *c, VtBlock *b, VtFile *p, u32int offset, int mode)
{
int epb;
- u32int size;
VtEntry e;
VtFile *r;
t@@ -74,15 +73,9 @@ vtfilealloc(VtCache *c, VtBlock *b, VtFile *p, u32int offset, int mode)
return nil;
}
- size = vtcacheblocksize(c);
- if(e.dsize > size || e.psize > size){
- werrstr("block sizes %ud, %ud bigger than cache block size %ud",
- e.psize, e.dsize, size);
- return nil;
- }
-
r = vtmallocz(sizeof(VtFile));
r->c = c;
+ r->bsize = b->size;
r->mode = mode;
r->dsize = e.dsize;
r->psize = e.psize;
t@@ -126,7 +119,7 @@ vtfileopenroot(VtCache *c, VtEntry *e)
VtBlock *b;
VtFile *f;
- b = vtcacheallocblock(c, VtDirType);
+ b = vtcacheallocblock(c, VtDirType, VtEntrySize);
if(b == nil)
return nil;
t@@ -191,8 +184,6 @@ _vtfilecreate(VtFile *r, int o, int psize, int dsize, int type)
u32int offset;
assert(ISLOCKED(r));
- assert(psize <= VtMaxLumpSize);
- assert(dsize <= VtMaxLumpSize);
assert(type == VtDirType || type == VtDataType);
if(!r->dir){
t@@ -325,12 +316,12 @@ vtfilegetsize(VtFile *r)
static int
shrinksize(VtFile *r, VtEntry *e, uvlong size)
{
- int i, depth, type, isdir, ppb;
+ int i, depth, bsiz, type, isdir, ppb;
uvlong ptrsz;
uchar score[VtScoreSize];
VtBlock *b;
- b = vtcacheglobal(r->c, e->score, e->type);
+ b = vtcacheglobal(r->c, e->score, e->type, r->dsize);
if(b == nil)
return -1;
t@@ -369,7 +360,11 @@ shrinksize(VtFile *r, VtEntry *e, uvlong size)
type--;
memmove(score, b->data+i*VtScoreSize, VtScoreSize);
vtblockput(b);
- b = vtcacheglobal(r->c, score, type);
+ if(type == VtDataType || type == VtDirType)
+ bsiz = r->dsize;
+ else
+ bsiz = r->psize;
+ b = vtcacheglobal(r->c, score, type, bsiz);
if(b == nil)
return -1;
}
t@@ -498,10 +493,10 @@ vtfilesetentry(VtFile *r, VtEntry *e)
}
static VtBlock *
-blockwalk(VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
+blockwalk(VtFile *r, VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
{
VtBlock *b;
- int type;
+ int type, size;
uchar *score;
VtEntry oe;
t@@ -519,12 +514,16 @@ blockwalk(VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
}
/*print("walk from %V/%d ty %d to %V ty %d\n", p->score, index, p->type, score, type); */
+ if(type == VtDirType || type == VtDataType)
+ size = r->dsize;
+ else
+ size = r->psize;
if(mode == VtOWRITE && vtglobaltolocal(score) == NilBlock){
- b = vtcacheallocblock(c, type);
+ b = vtcacheallocblock(c, type, size);
if(b)
goto HaveCopy;
}else
- b = vtcacheglobal(c, score, type);
+ b = vtcacheglobal(c, score, type, size);
if(b == nil || mode == VtOREAD)
return b;
t@@ -566,7 +565,7 @@ growdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
assert(ISLOCKED(r));
assert(depth <= VtPointerDepth);
- b = vtcacheglobal(r->c, e->score, e->type);
+ b = vtcacheglobal(r->c, e->score, e->type, r->dsize);
if(b == nil)
return -1;
t@@ -577,7 +576,7 @@ growdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
* or an error occurs.
*/
while(DEPTH(e->type) < depth){
- bb = vtcacheallocblock(r->c, e->type+1);
+ bb = vtcacheallocblock(r->c, e->type+1, r->psize);
if(bb == nil)
break;
memmove(bb->data, b->score, VtScoreSize);
t@@ -605,7 +604,7 @@ shrinkdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
assert(ISLOCKED(r));
assert(depth <= VtPointerDepth);
- rb = vtcacheglobal(r->c, e->score, e->type);
+ rb = vtcacheglobal(r->c, e->score, e->type, r->psize);
if(rb == nil)
return -1;
t@@ -618,7 +617,7 @@ shrinkdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
ob = nil;
b = rb;
for(; DEPTH(e->type) > depth; e->type--){
- nb = vtcacheglobal(r->c, b->data, e->type-1);
+ nb = vtcacheglobal(r->c, b->data, e->type-1, r->psize);
if(nb == nil)
break;
if(ob!=nil && ob!=rb)
t@@ -720,7 +719,7 @@ assert(b->type == VtDirType);
m = VtORDWR;
for(i=DEPTH(e.type); i>=0; i--){
- bb = blockwalk(b, index[i], r->c, i==0 ? mode : m, &e);
+ bb = blockwalk(r, b, index[i], r->c, i==0 ? mode : m, &e);
if(bb == nil)
goto Err;
vtblockput(b);
t@@ -768,7 +767,7 @@ vtfileblockscore(VtFile *r, u32int bn, uchar score[VtScoreSize])
index[DEPTH(e.type)] = r->offset % r->epb;
for(i=DEPTH(e.type); i>=1; i--){
- bb = blockwalk(b, index[i], r->c, VtOREAD, &e);
+ bb = blockwalk(r, b, index[i], r->c, VtOREAD, &e);
if(bb == nil)
goto Err;
vtblockput(b);
t@@ -837,7 +836,7 @@ fileloadblock(VtFile *r, int mode)
case VtORDWR:
assert(r->mode == VtORDWR);
if(r->local == 1){
- b = vtcacheglobal(r->c, r->score, VtDirType);
+ b = vtcacheglobal(r->c, r->score, VtDirType, r->bsize);
if(b == nil)
return nil;
b->pc = getcallerpc(&r);
t@@ -861,7 +860,7 @@ fileloadblock(VtFile *r, int mode)
}
addr = vtglobaltolocal(r->score);
if(addr == NilBlock)
- return vtcacheglobal(r->c, r->score, VtDirType);
+ return vtcacheglobal(r->c, r->score, VtDirType, r->bsize);
b = vtcachelocal(r->c, addr, VtDirType);
if(b)
t@@ -1220,7 +1219,7 @@ vtfileflushbefore(VtFile *r, u64int offset)
*/
index[depth] = r->offset % r->epb;
for(i=depth; i>=0; i--){
- bb = blockwalk(b, index[i], r->c, VtORDWR, &e);
+ bb = blockwalk(r, b, index[i], r->c, VtORDWR, &e);
if(bb == nil)
goto Err;
bi[i] = bb;
(DIR) diff --git a/src/libventi/root.c b/src/libventi/root.c
t@@ -6,19 +6,30 @@
static int
checksize(int n)
{
- if(n < 256 || n > VtMaxLumpSize) {
+ if(n < 256) {
werrstr("bad block size");
return -1;
}
return 0;
}
+extern int vttobig(ulong);
+
void
vtrootpack(VtRoot *r, uchar *p)
{
uchar *op = p;
+ int vers, bsize;
- U16PUT(p, VtRootVersion);
+ vers = VtRootVersion;
+ bsize = r->blocksize;
+ if(bsize >= (1<<16)) {
+ vers |= _VtRootVersionBig;
+ bsize = vttobig(bsize);
+ if(bsize < 0)
+ sysfatal("invalid root blocksize: %#x", r->blocksize);
+ }
+ U16PUT(p, vers);
p += 2;
memmove(p, r->name, sizeof(r->name));
p += sizeof(r->name);
t@@ -26,7 +37,7 @@ vtrootpack(VtRoot *r, uchar *p)
p += sizeof(r->type);
memmove(p, r->score, VtScoreSize);
p += VtScoreSize;
- U16PUT(p, r->blocksize);
+ U16PUT(p, bsize);
p += 2;
memmove(p, r->prev, VtScoreSize);
p += VtScoreSize;
t@@ -42,7 +53,7 @@ vtrootunpack(VtRoot *r, uchar *p)
memset(r, 0, sizeof(*r));
vers = U16GET(p);
- if(vers != VtRootVersion) {
+ if((vers&~_VtRootVersionBig) != VtRootVersion) {
werrstr("unknown root version");
return -1;
}
t@@ -56,6 +67,8 @@ vtrootunpack(VtRoot *r, uchar *p)
memmove(r->score, p, VtScoreSize);
p += VtScoreSize;
r->blocksize = U16GET(p);
+ if(vers & _VtRootVersionBig)
+ r->blocksize = (r->blocksize >> 5) << (r->blocksize & 31);
if(checksize(r->blocksize) < 0)
return -1;
p += 2;