diff --git a/SConstruct b/SConstruct index fef8d65..6cce775 100644 --- a/SConstruct +++ b/SConstruct @@ -152,6 +152,12 @@ SConscript('sys/SConscript', variant_dir='build/sys') SConscript('lib/libc/SConscript', variant_dir='build/lib/libc') SConscript('sbin/init/SConscript', variant_dir='build/sbin/init') +# Build Tools +env["TOOLCHAINBUILD"] = "TRUE" +env["CC"] = "cc" +env["CXX"] = "c++" +SConscript('sbin/newfs_o2fs/SConscript', variant_dir='build/tools/newfs_o2fs') + # Install Targets env.Install('$PREFIX/','build/sys/castor') env.Alias('install','$PREFIX') diff --git a/sbin/newfs_o2fs/SConscript b/sbin/newfs_o2fs/SConscript new file mode 100644 index 0000000..b9d358a --- /dev/null +++ b/sbin/newfs_o2fs/SConscript @@ -0,0 +1,19 @@ +import sys + +Import('env') + +newfs_env = env.Clone() + +src = [ + "newfs_o2fs.c" +] + +if newfs_env["TOOLCHAINBUILD"] != "TRUE": + newfs_env.Append(LINKFLAGS = ['-nostdlib']) + newfs_env.Append(CPPFLAGS = ['-nostdinc']) + newfs_env.Append(LIBPATH = ['#build/lib/libc'], LIBS = ['c']) + newfs_env.Append(CPPPATH = ['#build/include']) + +newfs_env.Append(CPPPATH = ['#sys/fs/o2fs']) +newfs_env.Program("newfs_o2fs", src) + diff --git a/sbin/newfs_o2fs/newfs_o2fs.c b/sbin/newfs_o2fs/newfs_o2fs.c new file mode 100644 index 0000000..0adf693 --- /dev/null +++ b/sbin/newfs_o2fs/newfs_o2fs.c @@ -0,0 +1,362 @@ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#define MAXBLOCKSIZE (64*1024*1024) + +char tempbuf[MAXBLOCKSIZE]; +char zerobuf[MAXBLOCKSIZE]; + +bool verbose = false; +bool hasManifest = false; +uint64_t diskSize = 0; +uint64_t diskOffset = 0; +uint64_t blockSize = 16*1024; +int diskfd; +struct stat diskstat; + +#define TOKEN_EOF 0 +#define TOKEN_DIR 1 +#define TOKEN_END 2 +#define TOKEN_FILE 3 +#define TOKEN_STRING 4 + +char *tokenBuf; +char *tokenCur; +char tokenString[512]; + +void LoadManifest(const char *manifest) +{ + int fd = open(manifest, O_RDONLY); + struct stat manifeststat; + + if (fd < 0) { + perror("Cannot open manifest"); + exit(1); + } + + fstat(fd, &manifeststat); + tokenBuf = malloc(manifeststat.st_size + 1); + read(fd, tokenBuf, manifeststat.st_size); + tokenBuf[manifeststat.st_size] = '\0'; + + tokenCur = tokenBuf; + tokenString[0] = '\0'; +} + +int GetToken() +{ + int i; + + while (*tokenCur == ' ' || *tokenCur == '\t' || + *tokenCur == '\n' || *tokenCur == '\r') + tokenCur++; + + for (i = 0; i < 512; i++) + { + tokenString[i] = tokenCur[i]; + if (tokenCur[i] == ' ' || tokenCur[i] == '\t' || + tokenCur[i] == '\n' || tokenCur[i] == '\r' || + tokenCur[i] == '\0') + { + tokenString[i] = '\0'; + tokenCur += i; + break; + } + } + + if (strcmp(tokenString, "") == 0) + return TOKEN_EOF; + if (strcmp(tokenString, "DIR") == 0) + return TOKEN_DIR; + if (strcmp(tokenString, "END") == 0) + return TOKEN_END; + if (strcmp(tokenString, "FILE") == 0) + return TOKEN_FILE; + return TOKEN_STRING; +} + +uint64_t +AppendBlock(const void *buf, size_t len) +{ + uint64_t offset = lseek(diskfd, 0, SEEK_CUR); + + assert(offset % blockSize == 0); + assert(len <= blockSize); + + write(diskfd, buf, len); + if (len != blockSize) { + write(diskfd, zerobuf, blockSize - len); + } + + return offset; +} + +ObjID *AddFile(const char *file) +{ + int i = 0; + int fd; + ObjID *id = malloc(sizeof(ObjID)); + BNode node; + + memset(&node, 0, sizeof(node)); + memcpy(node.magic, BNODE_MAGIC, 8); + node.versionMajor = O2FS_VERSION_MAJOR; + node.versionMinor = O2FS_VERSION_MINOR; + + // Copy file + fd = open(file, O_RDONLY); + if (fd < 0) { + perror("Cannot open file"); + exit(1); + } + while (1) { + int len = read(fd, tempbuf, blockSize); + if (len < 0) { + perror("File read error"); + exit(1); + } + if (len == 0) { + break; + } + + uint64_t offset = AppendBlock(tempbuf, len); + + // Update node + node.direct[i].device = 0; + node.direct[i].offset = offset; + node.size += (uint64_t)len; + i++; + } + close(fd); + + // Construct BNode + uint64_t offset = AppendBlock(&node, sizeof(node)); + + // Construct ObjID + id->device = 0; + id->offset = offset; + + return id; +} + +ObjID *AddDirectory() +{ + int tok; + BDirEntry *entries = malloc(128 * sizeof(BDirEntry)); + int entry = 0; + ObjID *id = malloc(sizeof(ObjID)); + BNode node; + + while (1) + { + memset(&entries[entry], 0, sizeof(BDirEntry)); + tok = GetToken(); + if (tok == TOKEN_FILE) { + tok = GetToken(); + printf("FILE %s\n", tokenString); + strncpy((char *)entries[entry].name, tokenString, MAXNAMELEN); + + tok = GetToken(); + ObjID *fobj = AddFile(tokenString); + memcpy(&entries[entry].objId, fobj, sizeof(ObjID)); + free(fobj); + } else if (tok == TOKEN_DIR) { + tok = GetToken(); + printf("DIR %s\n", tokenString); + strncpy((char *)entries[entry].name, tokenString, MAXNAMELEN); + ObjID *dobj = AddDirectory(); + memcpy(&entries[entry].objId, dobj, sizeof(ObjID)); + free(dobj); + } else if (tok == TOKEN_END) { + printf("END\n"); + break; + } else if (tok == TOKEN_EOF) { + printf("Unexpected end of file\n"); + exit(1); + } else { + printf("Unknown token '%s'\n", tokenString); + exit(1); + } + + memcpy(entries[entry].magic, BDIR_MAGIC, 8); + // entries[entry].size ... + entries[entry].flags = 0; + entries[entry].ctime = (uint64_t)time(NULL); + entries[entry].mtime = (uint64_t)time(NULL); + strncpy((char *)entries[entry].user, "root", MAXUSERNAMELEN); + strncpy((char *)entries[entry].group, "root", MAXUSERNAMELEN); + + entry++; + } + + // Write Directory + uint64_t dirOffset = AppendBlock(entries, entry * sizeof(BDirEntry)); + uint64_t dirLength = entry * sizeof(BDirEntry); + int i; + uint64_t off; + + memset(&node, 0, sizeof(node)); + memcpy(node.magic, BNODE_MAGIC, 8); + node.versionMajor = O2FS_VERSION_MAJOR; + node.versionMinor = O2FS_VERSION_MINOR; + node.size = dirLength; + + for (i = 0, off = 0; off <= dirLength; i++, off += blockSize) { + node.direct[i].device = 0; + node.direct[i].offset = dirOffset + off; + } + + uint64_t offset = AppendBlock(&node, sizeof(node)); + + id->device = 0; + id->offset = offset; + + return id; +} + +void Superblock(ObjID *objid) +{ + SuperBlock sb; + + lseek(diskfd, 0, SEEK_SET); + + memset(&sb, 0, sizeof(sb)); + memcpy(sb.magic, SUPERBLOCK_MAGIC, 8); + sb.versionMajor = O2FS_VERSION_MAJOR; + sb.versionMinor = O2FS_VERSION_MINOR; + sb.blockCount = diskSize / blockSize; + sb.blockSize = blockSize; + + if (objid) + memcpy(&sb.root, objid, sizeof(ObjID)); + + AppendBlock(&sb, sizeof(sb)); +} + +void usage() +{ + printf("Usage: newfs_o2fs [OPTIONS] special-device\n"); + printf("Options:\n"); + printf(" -m, --manifest Manifest of files to copy to file system\n"); + printf(" -s, --size Size in megabytes of device or disk image\n"); + printf(" -v, --verbose Verbose logging\n"); + printf(" -h, --help Print help message\n"); +} + +int main(int argc, char * const *argv) +{ + int ch; + int status; + + struct option longopts[] = { + { "manifest", required_argument, NULL, 'm' }, + { "size", required_argument, NULL, 's' }, + { "verbose", no_argument, NULL, 'v' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + while ((ch = getopt_long(argc, argv, "m:s:vh", longopts, NULL)) != -1) + { + switch (ch) { + case 'm': + hasManifest = true; + LoadManifest(optarg); + break; + case 's': + diskSize = atol(optarg) * 1024 * 1024; + break; + case 'v': + verbose = true; + break; + case 'h': + usage(); + return 0; + default: + usage(); + return 1; + } + } + + argc -= optind; + argv += optind; + + if (argc != 1) { + usage(); + return 1; + } + + diskfd = open(argv[0], O_RDWR | O_CREAT, 0660); + if (diskfd < 0) { + perror("Cannot open special device or disk image"); + return 1; + } + + status = fstat(diskfd, &diskstat); + if (status < 0) { + perror("Cannot fstat special device or disk image"); + return 1; + } + + if (diskstat.st_size == 0 && diskSize == 0) { + printf("Error: Must specify size for disk images\n"); + usage(); + return 1; + } + + if (diskstat.st_size == 0) { + char buf[512]; + + lseek(diskfd, diskSize - 512, SEEK_SET); + memset(buf, 0, 512); + write(diskfd, buf, 512); + lseek(diskfd, 0, SEEK_SET); + } + + diskOffset = blockSize; + lseek(diskfd, diskOffset, SEEK_SET); + memset(zerobuf, 0, MAXBLOCKSIZE); + + ObjID *root = NULL; + if (hasManifest) { + int tok; + + tok = GetToken(); + if (tok != TOKEN_DIR) { + printf("Expected 'DIR' token, but found '%s'\n", tokenString); + exit(1); + } + tok = GetToken(); + if (tok != TOKEN_STRING || strcmp(tokenString, "/") != 0) { + printf("Expected '/' token\n"); + exit(1); + } + + root = AddDirectory(); + tok = GetToken(); + if (tok != TOKEN_EOF) { + printf("Expected end-of-file, but found '%s'\n", tokenString); + exit(1); + } + } + + Superblock(root); + free(root); + + close(diskfd); +} + diff --git a/sys/SConscript b/sys/SConscript index 864b6d8..32c0f41 100644 --- a/sys/SConscript +++ b/sys/SConscript @@ -45,9 +45,11 @@ src_common = [ "kern/spinlock.c", "kern/syscall.c", "kern/thread.c", + "kern/vfs.c", "dev/ahci.c", "dev/console.c", "dev/pci.c", + "fs/o2fs/o2fs.c", ] if (env["ARCH"] == "amd64"): diff --git a/sys/amd64/machine.c b/sys/amd64/machine.c index 0384505..6ddc714 100644 --- a/sys/amd64/machine.c +++ b/sys/amd64/machine.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "../dev/console.h" @@ -140,6 +141,11 @@ void Machine_Init() IDE_Init(); DiskCache_Init(); + Disk *root = Disk_GetByID(0, 0); + if (!root) + Panic("No boot disk!"); + VFS_MountRoot(root); + Critical_Exit(); Thread *thr = Thread_KThreadCreate(&Machine_IdleThread, NULL); diff --git a/sys/fs/o2fs/o2fs.c b/sys/fs/o2fs/o2fs.c new file mode 100644 index 0000000..bfa97c1 --- /dev/null +++ b/sys/fs/o2fs/o2fs.c @@ -0,0 +1,159 @@ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "o2fs.h" + +VFS *O2FS_Mount(Disk *disk); +int O2FS_Unmount(VFS *fs); +int O2FS_GetRoot(VFS *fs, VNode **dn); +int O2FS_Lookup(VNode *dn, VNode **fn, const char *name); +int O2FS_Open(VNode *fn); +int O2FS_Close(VNode *fn); +int O2FS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len); + +static VFSOp O2FSOperations = { + .unmount = O2FS_Unmount, + .getroot = O2FS_GetRoot, + .lookup = O2FS_Lookup, + .open = O2FS_Open, + .close = O2FS_Close, + .read = O2FS_Read, +}; + +VFS * +O2FS_Mount(Disk *disk) +{ + int status; + VFS *fs = (VFS *)PAlloc_AllocPage(); + DiskCacheEntry *entry; + SuperBlock *sb; + + if (!fs) + return NULL; + + status = DiskCache_Read(disk, 0, &entry); + if (status < 0) { + kprintf("O2FS: Disk cache read failed\n"); + return NULL; + } + + Debug_PrintHex(entry->buffer, 512, 0, 512); + + // Read superblock + sb = entry->buffer; + if (memcmp(sb->magic, SUPERBLOCK_MAGIC, 8) != 0) { + kprintf("O2FS: Invalid file system\n"); + DiskCache_Release(entry); + return NULL; + } + if (sb->versionMajor != O2FS_VERSION_MAJOR || + sb->versionMinor != O2FS_VERSION_MINOR) { + kprintf("O2FS: Unsupported file system version\n"); + DiskCache_Release(entry); + return NULL; + } + + kprintf("O2FS: File system mounted\n"); + kprintf("O2FS: Root @ 0x%llx\n", sb->root.offset); + + fs->fsptr = entry; + fs->fsval = sb->root.offset; + + // Setup VFS structure + fs->op = &O2FSOperations; + fs->disk = disk; + Spinlock_Init(&fs->lock, "O2FS Lock"); + fs->refCount = 1; + fs->root = NULL; + + status = O2FS_GetRoot(fs, &fs->root); + if (status < 0) { + kprintf("O2FS: Mount failed"); + DiskCache_Release(entry); + return NULL; + } + + return fs; +} + +int +O2FS_Unmount(VFS *fs) +{ +} + +int +O2FS_GetRoot(VFS *fs, VNode **dn) +{ + int status; + VNode *vn; + DiskCacheEntry *entry; + BNode *bn; + + if (fs->root) { + fs->root->refCount++; + *dn = fs->root; + return 0; + } + + status = DiskCache_Read(fs->disk, fs->fsval, &entry); + if (status < 0) { + kprintf("O2FS: disk read error\n"); + return status; + } + + Debug_PrintHex(entry->buffer, 512, 0, 512); + + bn = entry->buffer; + if (memcmp(&bn->magic, BNODE_MAGIC, 8) != 0) { + kprintf("O2FS: bad BNode magic\n"); + DiskCache_Release(entry); + return -1; + } + if (bn->versionMajor != O2FS_VERSION_MAJOR || + bn->versionMinor != O2FS_VERSION_MINOR) { + kprintf("O2FS: unsupported BNode version\n"); + DiskCache_Release(entry); + return -1; + } + + vn = (VNode *)PAlloc_AllocPage(); + vn->op = &O2FSOperations; + vn->disk = fs->disk; + Spinlock_Init(&vn->lock, "VNode Lock"); + vn->refCount = 1; + vn->fsptr = entry; + + *dn = vn; + + return 0; +} + +int +O2FS_Lookup(VNode *dn, VNode **fn, const char *name) +{ +} + +int +O2FS_Open(VNode *fn) +{ +} + +int +O2FS_Close(VNode *fn) +{ +} + +int +O2FS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len) +{ +} + diff --git a/sys/fs/o2fs/o2fs.h b/sys/fs/o2fs/o2fs.h index de56455..583bb76 100644 --- a/sys/fs/o2fs/o2fs.h +++ b/sys/fs/o2fs/o2fs.h @@ -1,4 +1,7 @@ +#ifndef __FS_O2FS_H__ +#define __FS_O2FS_H__ + /* * *** Disk Layout *** * @@ -16,12 +19,29 @@ * */ +#define MAXUSERNAMELEN 32 +#define MAXNAMELEN 255 + +/* + * *** Version History *** + * 1.0 Initial release + */ + +#define O2FS_VERSION_MAJOR 1 +#define O2FS_VERSION_MINOR 0 + +/* + * Object ID: Pointer to a block node + */ typedef struct ObjID { uint8_t hash[32]; uint64_t device; uint64_t offset; } ObjID; +/* + * Super block + */ typedef struct SuperBlock { uint8_t magic[8]; @@ -29,8 +49,61 @@ typedef struct SuperBlock uint16_t versionMinor; uint32_t _rsvd0; uint64_t features; + uint64_t blockCount; + uint64_t blockSize; ObjID root; /* Root Tree */ uint8_t hash[32]; } SuperBlock; + +#define SUPERBLOCK_MAGIC "SUPRBLOK" + +/* + * Block Pointer: Address raw blocks on the disk + */ +typedef struct BPtr { + uint8_t hash[32]; + uint64_t device; + uint64_t offset; + uint64_t _rsvd0; + uint64_t _rsvd1; +} BPtr; + +/* + * Block Nodes: Contain indirect pointers to pieces of a file + */ +typedef struct BNode +{ + uint8_t magic[8]; + uint16_t versionMajor; + uint16_t versionMinor; + uint32_t flags; + uint64_t size; + + BPtr direct[64]; +} BNode; + +#define BNODE_MAGIC "BLOKNODE" + +/* + * Directory entries are exactly 512 bytes + */ +typedef struct BDirEntry +{ + uint8_t magic[8]; + ObjID objId; + uint64_t size; + uint64_t flags; + uint64_t ctime; + uint64_t mtime; + uint8_t user[MAXUSERNAMELEN]; // Not null terminated + uint8_t group[MAXUSERNAMELEN]; + uint8_t padding[104]; + uint8_t name[MAXNAMELEN+1]; // Null terminated +} BDirEntry; + +#define BDIR_MAGIC "DIRENTRY" + +#endif /* __FS_O2FS_H__ */ + diff --git a/sys/include/vfs.h b/sys/include/vfs.h new file mode 100644 index 0000000..d2a4b52 --- /dev/null +++ b/sys/include/vfs.h @@ -0,0 +1,47 @@ + +#ifndef __SYS_VFS_H__ +#define __SYS_VFS_H__ + +typedef struct VFSOp VFSOp; +typedef struct VNode VNode; + +typedef struct VFS { + VFSOp *op; + Disk *disk; + Spinlock lock; + uint64_t refCount; + // FS Fields + void *fsptr; + uint64_t fsval; + VNode *root; +} VFS; + +typedef struct VNode { + VFSOp *op; + Disk *disk; + Spinlock lock; + uint64_t refCount; + // FS Fields + void *fsptr; + uint64_t fsval; +} VNode; + +typedef struct VFSOp { + // VFS Operations + int (*unmount)(VFS *fs); + int (*getroot)(VFS *fs, VNode **dn); + // VNode Operations + int (*lookup)(VNode *dn, VNode **fn, const char *name); + int (*open)(VNode *fn); + int (*close)(VNode *fn); + int (*read)(VNode *fn, void *buf, uint64_t off, uint64_t len); +} VFSOp; + +int VFS_MountRoot(Disk *root); +VNode *VFS_Lookup(const char *path); +int VFS_Open(VNode *fn); +int VFS_Close(VNode *fn); +int VFS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len); + +#endif /* __SYS_VFS_H__ */ + diff --git a/sys/kern/diskcache.c b/sys/kern/diskcache.c index 6f143af..afad64f 100644 --- a/sys/kern/diskcache.c +++ b/sys/kern/diskcache.c @@ -15,6 +15,9 @@ XMem *diskBuf; static TAILQ_HEAD(CacheHashTable, DiskCacheEntry) *hashTable; static TAILQ_HEAD(LRUCacheList, DiskCacheEntry) lruList; +static uint64_t cacheHit; +static uint64_t cacheMiss; +static uint64_t cacheAlloc; #define CACHESIZE (16*1024*1024) #define HASHTABLEENTRIES 128 @@ -55,6 +58,10 @@ DiskCache_Init() e->buffer = (void *)(bufBase + BLOCKSIZE * i); TAILQ_INSERT_TAIL(&lruList, e, lruEntry); } + + cacheHit = 0; + cacheMiss = 0; + cacheAlloc = 0; } int @@ -125,6 +132,8 @@ DiskCache_Alloc(Disk *disk, uint64_t diskOffset, DiskCacheEntry **entry) status = DiskCacheAlloc(disk, diskOffset, entry); } + cacheAlloc++; + Spinlock_Unlock(&cacheLock); return status; @@ -153,9 +162,12 @@ DiskCache_Read(Disk *disk, uint64_t diskOffset, DiskCacheEntry **entry) Spinlock_Lock(&cacheLock); status = DiskCacheLookup(disk, diskOffset, entry); if (*entry != NULL) { + cacheHit++; Spinlock_Unlock(&cacheLock); return status; } + cacheMiss++; + status = DiskCacheAlloc(disk, diskOffset, entry); if (status != 0) { Spinlock_Unlock(&cacheLock); @@ -188,3 +200,13 @@ DiskCache_Write(DiskCacheEntry *entry) return Disk_Write(entry->disk, buf, &sga, NULL, NULL); } +void +Debug_DiskCache(int argc, const char *argv[]) +{ + kprintf("Hits: %lld\n", cacheHit); + kprintf("Misses: %lld\n", cacheMiss); + kprintf("Allocations: %lld\n", cacheAlloc); +} + +REGISTER_DBGCMD(diskcache, "Display disk cache statistics", Debug_DiskCache); + diff --git a/sys/kern/vfs.c b/sys/kern/vfs.c new file mode 100644 index 0000000..7d70003 --- /dev/null +++ b/sys/kern/vfs.c @@ -0,0 +1,101 @@ + +#include +#include +#include + +#include +#include +#include +#include +#include + +extern VFS *O2FS_Mount(Disk *root); + +static Spinlock vfsLock; +static VFS *rootFS; +static VNode *rootNode; + +int +VFS_MountRoot(Disk *rootDisk) +{ + int status; + + Spinlock_Init(&vfsLock, "VFS Lock"); + + rootFS = O2FS_Mount(rootDisk); + if (!rootFS) + return -1; + + status = rootFS->op->getroot(rootFS, &rootNode); + if (status < 0) + Panic("Failed to get root VNode\n"); + + return 0; +} + +VNode * +VFS_Lookup(const char *path) +{ + int status; + const char *start = path + 1; + const char *end = path + 1; + uint64_t len; + VNode *curNode; + VNode *oldNode; + char curName[256]; + + if (path[0] != '/') + return NULL; + + status = rootFS->op->getroot(rootFS, &curNode); + if (status < 0) + Panic("Failed to get root VNode\n"); + + while (1) { + while (*end != '\0' || *end != '/') + end++; + + len = (size_t)(end - start); + if (len == 0 || len > 256) { + // Release + return NULL; + } + + memcpy(curName, start, len); + curName[len - 1] = '\0'; + + oldNode = curNode; + status = curNode->op->lookup(curNode, &curNode, curName); + if (status < 0) { + // Release + return NULL; + } + + // Release oldNode + + if (*end == '\0') + return curNode; + + start = end + 1; + end = end + 1; + } +} + +int +VFS_Open(VNode *fn) +{ + return fn->op->open(fn); +} + +int +VFS_Close(VNode *fn) +{ + return fn->op->close(fn); +} + +int +VFS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len) +{ + return fn->op->read(fn, buf, off, len); +} +