d/metal-cos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Beginning O2FS and VFS code along with superblock and root directory reading.

Browse Source
This commit is contained in:
Ali Mashtizadeh 2014-08-01 14:15:06 -07:00
parent 3e35c771cc
commit 2a79939b79
10 changed files with 797 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
SConstruct
View File

@ -152,6 +152,12 @@ SConscript('sys/SConscript', variant_dir='build/sys')
SConscript('lib/libc/SConscript', variant_dir='build/lib/libc')
SConscript('sbin/init/SConscript', variant_dir='build/sbin/init')
# Build Tools
env["TOOLCHAINBUILD"] = "TRUE"
env["CC"] = "cc"
env["CXX"] = "c++"
SConscript('sbin/newfs_o2fs/SConscript', variant_dir='build/tools/newfs_o2fs')
# Install Targets
env.Install('$PREFIX/','build/sys/castor')
env.Alias('install','$PREFIX')

19
sbin/newfs_o2fs/SConscript Normal file
View File

@ -0,0 +1,19 @@
import sys
Import('env')
newfs_env = env.Clone()
src = [
"newfs_o2fs.c"
]
if newfs_env["TOOLCHAINBUILD"] != "TRUE":
newfs_env.Append(LINKFLAGS = ['-nostdlib'])
newfs_env.Append(CPPFLAGS = ['-nostdinc'])
newfs_env.Append(LIBPATH = ['#build/lib/libc'], LIBS = ['c'])
newfs_env.Append(CPPPATH = ['#build/include'])
newfs_env.Append(CPPPATH = ['#sys/fs/o2fs'])
newfs_env.Program("newfs_o2fs", src)

362
sbin/newfs_o2fs/newfs_o2fs.c Normal file
View File

@ -0,0 +1,362 @@
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <time.h>
#include <getopt.h>
#include <o2fs.h>
#define MAXBLOCKSIZE (64*1024*1024)
char tempbuf[MAXBLOCKSIZE];
char zerobuf[MAXBLOCKSIZE];
bool verbose = false;
bool hasManifest = false;
uint64_t diskSize = 0;
uint64_t diskOffset = 0;
uint64_t blockSize = 16*1024;
int diskfd;
struct stat diskstat;
#define TOKEN_EOF 0
#define TOKEN_DIR 1
#define TOKEN_END 2
#define TOKEN_FILE 3
#define TOKEN_STRING 4
char *tokenBuf;
char *tokenCur;
char tokenString[512];
void LoadManifest(const char *manifest)
{
int fd = open(manifest, O_RDONLY);
struct stat manifeststat;
if (fd < 0) {
perror("Cannot open manifest");
exit(1);
}
fstat(fd, &manifeststat);
tokenBuf = malloc(manifeststat.st_size + 1);
read(fd, tokenBuf, manifeststat.st_size);
tokenBuf[manifeststat.st_size] = '\0';
tokenCur = tokenBuf;
tokenString[0] = '\0';
}
int GetToken()
{
int i;
while (*tokenCur == ' ' || *tokenCur == '\t' ||
*tokenCur == '\n' || *tokenCur == '\r')
tokenCur++;
for (i = 0; i < 512; i++)
{
tokenString[i] = tokenCur[i];
if (tokenCur[i] == ' ' || tokenCur[i] == '\t' ||
tokenCur[i] == '\n' || tokenCur[i] == '\r' ||
tokenCur[i] == '\0')
{
tokenString[i] = '\0';
tokenCur += i;
break;
}
}
if (strcmp(tokenString, "") == 0)
return TOKEN_EOF;
if (strcmp(tokenString, "DIR") == 0)
return TOKEN_DIR;
if (strcmp(tokenString, "END") == 0)
return TOKEN_END;
if (strcmp(tokenString, "FILE") == 0)
return TOKEN_FILE;
return TOKEN_STRING;
}
uint64_t
AppendBlock(const void *buf, size_t len)
{
uint64_t offset = lseek(diskfd, 0, SEEK_CUR);
assert(offset % blockSize == 0);
assert(len <= blockSize);
write(diskfd, buf, len);
if (len != blockSize) {
write(diskfd, zerobuf, blockSize - len);
}
return offset;
}
ObjID *AddFile(const char *file)
{
int i = 0;
int fd;
ObjID *id = malloc(sizeof(ObjID));
BNode node;
memset(&node, 0, sizeof(node));
memcpy(node.magic, BNODE_MAGIC, 8);
node.versionMajor = O2FS_VERSION_MAJOR;
node.versionMinor = O2FS_VERSION_MINOR;
// Copy file
fd = open(file, O_RDONLY);
if (fd < 0) {
perror("Cannot open file");
exit(1);
}
while (1) {
int len = read(fd, tempbuf, blockSize);
if (len < 0) {
perror("File read error");
exit(1);
}
if (len == 0) {
break;
}
uint64_t offset = AppendBlock(tempbuf, len);
// Update node
node.direct[i].device = 0;
node.direct[i].offset = offset;
node.size += (uint64_t)len;
i++;
}
close(fd);
// Construct BNode
uint64_t offset = AppendBlock(&node, sizeof(node));
// Construct ObjID
id->device = 0;
id->offset = offset;
return id;
}
ObjID *AddDirectory()
{
int tok;
BDirEntry *entries = malloc(128 * sizeof(BDirEntry));
int entry = 0;
ObjID *id = malloc(sizeof(ObjID));
BNode node;
while (1)
{
memset(&entries[entry], 0, sizeof(BDirEntry));
tok = GetToken();
if (tok == TOKEN_FILE) {
tok = GetToken();
printf("FILE %s\n", tokenString);
strncpy((char *)entries[entry].name, tokenString, MAXNAMELEN);
tok = GetToken();
ObjID *fobj = AddFile(tokenString);
memcpy(&entries[entry].objId, fobj, sizeof(ObjID));
free(fobj);
} else if (tok == TOKEN_DIR) {
tok = GetToken();
printf("DIR %s\n", tokenString);
strncpy((char *)entries[entry].name, tokenString, MAXNAMELEN);
ObjID *dobj = AddDirectory();
memcpy(&entries[entry].objId, dobj, sizeof(ObjID));
free(dobj);
} else if (tok == TOKEN_END) {
printf("END\n");
break;
} else if (tok == TOKEN_EOF) {
printf("Unexpected end of file\n");
exit(1);
} else {
printf("Unknown token '%s'\n", tokenString);
exit(1);
}
memcpy(entries[entry].magic, BDIR_MAGIC, 8);
// entries[entry].size ...
entries[entry].flags = 0;
entries[entry].ctime = (uint64_t)time(NULL);
entries[entry].mtime = (uint64_t)time(NULL);
strncpy((char *)entries[entry].user, "root", MAXUSERNAMELEN);
strncpy((char *)entries[entry].group, "root", MAXUSERNAMELEN);
entry++;
}
// Write Directory
uint64_t dirOffset = AppendBlock(entries, entry * sizeof(BDirEntry));
uint64_t dirLength = entry * sizeof(BDirEntry);
int i;
uint64_t off;
memset(&node, 0, sizeof(node));
memcpy(node.magic, BNODE_MAGIC, 8);
node.versionMajor = O2FS_VERSION_MAJOR;
node.versionMinor = O2FS_VERSION_MINOR;
node.size = dirLength;
for (i = 0, off = 0; off <= dirLength; i++, off += blockSize) {
node.direct[i].device = 0;
node.direct[i].offset = dirOffset + off;
}
uint64_t offset = AppendBlock(&node, sizeof(node));
id->device = 0;
id->offset = offset;
return id;
}
void Superblock(ObjID *objid)
{
SuperBlock sb;
lseek(diskfd, 0, SEEK_SET);
memset(&sb, 0, sizeof(sb));
memcpy(sb.magic, SUPERBLOCK_MAGIC, 8);
sb.versionMajor = O2FS_VERSION_MAJOR;
sb.versionMinor = O2FS_VERSION_MINOR;
sb.blockCount = diskSize / blockSize;
sb.blockSize = blockSize;
if (objid)
memcpy(&sb.root, objid, sizeof(ObjID));
AppendBlock(&sb, sizeof(sb));
}
void usage()
{
printf("Usage: newfs_o2fs [OPTIONS] special-device\n");
printf("Options:\n");
printf(" -m, --manifest Manifest of files to copy to file system\n");
printf(" -s, --size Size in megabytes of device or disk image\n");
printf(" -v, --verbose Verbose logging\n");
printf(" -h, --help Print help message\n");
}
int main(int argc, char * const *argv)
{
int ch;
int status;
struct option longopts[] = {
{ "manifest", required_argument, NULL, 'm' },
{ "size", required_argument, NULL, 's' },
{ "verbose", no_argument, NULL, 'v' },
{ "help", no_argument, NULL, 'h' },
{ NULL, 0, NULL, 0 }
};
while ((ch = getopt_long(argc, argv, "m:s:vh", longopts, NULL)) != -1)
{
switch (ch) {
case 'm':
hasManifest = true;
LoadManifest(optarg);
break;
case 's':
diskSize = atol(optarg) * 1024 * 1024;
break;
case 'v':
verbose = true;
break;
case 'h':
usage();
return 0;
default:
usage();
return 1;
}
}
argc -= optind;
argv += optind;
if (argc != 1) {
usage();
return 1;
}
diskfd = open(argv[0], O_RDWR | O_CREAT, 0660);
if (diskfd < 0) {
perror("Cannot open special device or disk image");
return 1;
}
status = fstat(diskfd, &diskstat);
if (status < 0) {
perror("Cannot fstat special device or disk image");
return 1;
}
if (diskstat.st_size == 0 && diskSize == 0) {
printf("Error: Must specify size for disk images\n");
usage();
return 1;
}
if (diskstat.st_size == 0) {
char buf[512];
lseek(diskfd, diskSize - 512, SEEK_SET);
memset(buf, 0, 512);
write(diskfd, buf, 512);
lseek(diskfd, 0, SEEK_SET);
}
diskOffset = blockSize;
lseek(diskfd, diskOffset, SEEK_SET);
memset(zerobuf, 0, MAXBLOCKSIZE);
ObjID *root = NULL;
if (hasManifest) {
int tok;
tok = GetToken();
if (tok != TOKEN_DIR) {
printf("Expected 'DIR' token, but found '%s'\n", tokenString);
exit(1);
}
tok = GetToken();
if (tok != TOKEN_STRING || strcmp(tokenString, "/") != 0) {
printf("Expected '/' token\n");
exit(1);
}
root = AddDirectory();
tok = GetToken();
if (tok != TOKEN_EOF) {
printf("Expected end-of-file, but found '%s'\n", tokenString);
exit(1);
}
}
Superblock(root);
free(root);
close(diskfd);
}

2
sys/SConscript
View File

@ -45,9 +45,11 @@ src_common = [
"kern/spinlock.c",
"kern/syscall.c",
"kern/thread.c",
"kern/vfs.c",
"dev/ahci.c",
"dev/console.c",
"dev/pci.c",
"fs/o2fs/o2fs.c",
]
if (env["ARCH"] == "amd64"):

6
sys/amd64/machine.c
View File

@ -17,6 +17,7 @@
#include <sys/thread.h>
#include <sys/disk.h>
#include <sys/diskcache.h>
#include <sys/vfs.h>
#include <sys/elf64.h>
#include "../dev/console.h"
@ -140,6 +141,11 @@ void Machine_Init()
IDE_Init();
DiskCache_Init();
Disk *root = Disk_GetByID(0, 0);
if (!root)
Panic("No boot disk!");
VFS_MountRoot(root);
Critical_Exit();
Thread *thr = Thread_KThreadCreate(&Machine_IdleThread, NULL);

159
sys/fs/o2fs/o2fs.c Normal file
View File

@ -0,0 +1,159 @@
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <sys/kassert.h>
#include <sys/kmem.h>
#include <sys/spinlock.h>
#include <sys/disk.h>
#include <sys/diskcache.h>
#include <sys/vfs.h>
#include "o2fs.h"
VFS *O2FS_Mount(Disk *disk);
int O2FS_Unmount(VFS *fs);
int O2FS_GetRoot(VFS *fs, VNode **dn);
int O2FS_Lookup(VNode *dn, VNode **fn, const char *name);
int O2FS_Open(VNode *fn);
int O2FS_Close(VNode *fn);
int O2FS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len);
static VFSOp O2FSOperations = {
.unmount = O2FS_Unmount,
.getroot = O2FS_GetRoot,
.lookup = O2FS_Lookup,
.open = O2FS_Open,
.close = O2FS_Close,
.read = O2FS_Read,
};
VFS *
O2FS_Mount(Disk *disk)
{
int status;
VFS *fs = (VFS *)PAlloc_AllocPage();
DiskCacheEntry *entry;
SuperBlock *sb;
if (!fs)
return NULL;
status = DiskCache_Read(disk, 0, &entry);
if (status < 0) {
kprintf("O2FS: Disk cache read failed\n");
return NULL;
}
Debug_PrintHex(entry->buffer, 512, 0, 512);
// Read superblock
sb = entry->buffer;
if (memcmp(sb->magic, SUPERBLOCK_MAGIC, 8) != 0) {
kprintf("O2FS: Invalid file system\n");
DiskCache_Release(entry);
return NULL;
}
if (sb->versionMajor != O2FS_VERSION_MAJOR ||
sb->versionMinor != O2FS_VERSION_MINOR) {
kprintf("O2FS: Unsupported file system version\n");
DiskCache_Release(entry);
return NULL;
}
kprintf("O2FS: File system mounted\n");
kprintf("O2FS: Root @ 0x%llx\n", sb->root.offset);
fs->fsptr = entry;
fs->fsval = sb->root.offset;
// Setup VFS structure
fs->op = &O2FSOperations;
fs->disk = disk;
Spinlock_Init(&fs->lock, "O2FS Lock");
fs->refCount = 1;
fs->root = NULL;
status = O2FS_GetRoot(fs, &fs->root);
if (status < 0) {
kprintf("O2FS: Mount failed");
DiskCache_Release(entry);
return NULL;
}
return fs;
}
int
O2FS_Unmount(VFS *fs)
{
}
int
O2FS_GetRoot(VFS *fs, VNode **dn)
{
int status;
VNode *vn;
DiskCacheEntry *entry;
BNode *bn;
if (fs->root) {
fs->root->refCount++;
*dn = fs->root;
return 0;
}
status = DiskCache_Read(fs->disk, fs->fsval, &entry);
if (status < 0) {
kprintf("O2FS: disk read error\n");
return status;
}
Debug_PrintHex(entry->buffer, 512, 0, 512);
bn = entry->buffer;
if (memcmp(&bn->magic, BNODE_MAGIC, 8) != 0) {
kprintf("O2FS: bad BNode magic\n");
DiskCache_Release(entry);
return -1;
}
if (bn->versionMajor != O2FS_VERSION_MAJOR ||
bn->versionMinor != O2FS_VERSION_MINOR) {
kprintf("O2FS: unsupported BNode version\n");
DiskCache_Release(entry);
return -1;
}
vn = (VNode *)PAlloc_AllocPage();
vn->op = &O2FSOperations;
vn->disk = fs->disk;
Spinlock_Init(&vn->lock, "VNode Lock");
vn->refCount = 1;
vn->fsptr = entry;
*dn = vn;
return 0;
}
int
O2FS_Lookup(VNode *dn, VNode **fn, const char *name)
{
}
int
O2FS_Open(VNode *fn)
{
}
int
O2FS_Close(VNode *fn)
{
}
int
O2FS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len)
{
}

73
sys/fs/o2fs/o2fs.h
View File

@ -1,4 +1,7 @@
#ifndef __FS_O2FS_H__
#define __FS_O2FS_H__
/*
* *** Disk Layout ***
*
@ -16,12 +19,29 @@
*
*/
#define MAXUSERNAMELEN 32
#define MAXNAMELEN 255
/*
* *** Version History ***
* 1.0 Initial release
*/
#define O2FS_VERSION_MAJOR 1
#define O2FS_VERSION_MINOR 0
/*
* Object ID: Pointer to a block node
*/
typedef struct ObjID {
uint8_t hash[32];
uint64_t device;
uint64_t offset;
} ObjID;
/*
* Super block
*/
typedef struct SuperBlock
{
uint8_t magic[8];
@ -29,8 +49,61 @@ typedef struct SuperBlock
uint16_t versionMinor;
uint32_t _rsvd0;
uint64_t features;
uint64_t blockCount;
uint64_t blockSize;
ObjID root; /* Root Tree */
uint8_t hash[32];
} SuperBlock;
#define SUPERBLOCK_MAGIC "SUPRBLOK"
/*
* Block Pointer: Address raw blocks on the disk
*/
typedef struct BPtr {
uint8_t hash[32];
uint64_t device;
uint64_t offset;
uint64_t _rsvd0;
uint64_t _rsvd1;
} BPtr;
/*
* Block Nodes: Contain indirect pointers to pieces of a file
*/
typedef struct BNode
{
uint8_t magic[8];
uint16_t versionMajor;
uint16_t versionMinor;
uint32_t flags;
uint64_t size;
BPtr direct[64];
} BNode;
#define BNODE_MAGIC "BLOKNODE"
/*
* Directory entries are exactly 512 bytes
*/
typedef struct BDirEntry
{
uint8_t magic[8];
ObjID objId;
uint64_t size;
uint64_t flags;
uint64_t ctime;
uint64_t mtime;
uint8_t user[MAXUSERNAMELEN]; // Not null terminated
uint8_t group[MAXUSERNAMELEN];
uint8_t padding[104];
uint8_t name[MAXNAMELEN+1]; // Null terminated
} BDirEntry;
#define BDIR_MAGIC "DIRENTRY"
#endif /* __FS_O2FS_H__ */

47
sys/include/vfs.h Normal file
View File

@ -0,0 +1,47 @@
#ifndef __SYS_VFS_H__
#define __SYS_VFS_H__
typedef struct VFSOp VFSOp;
typedef struct VNode VNode;
typedef struct VFS {
VFSOp *op;
Disk *disk;
Spinlock lock;
uint64_t refCount;
// FS Fields
void *fsptr;
uint64_t fsval;
VNode *root;
} VFS;
typedef struct VNode {
VFSOp *op;
Disk *disk;
Spinlock lock;
uint64_t refCount;
// FS Fields
void *fsptr;
uint64_t fsval;
} VNode;
typedef struct VFSOp {
// VFS Operations
int (*unmount)(VFS *fs);
int (*getroot)(VFS *fs, VNode **dn);
// VNode Operations
int (*lookup)(VNode *dn, VNode **fn, const char *name);
int (*open)(VNode *fn);
int (*close)(VNode *fn);
int (*read)(VNode *fn, void *buf, uint64_t off, uint64_t len);
} VFSOp;
int VFS_MountRoot(Disk *root);
VNode *VFS_Lookup(const char *path);
int VFS_Open(VNode *fn);
int VFS_Close(VNode *fn);
int VFS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len);
#endif /* __SYS_VFS_H__ */

22
sys/kern/diskcache.c
View File

@ -15,6 +15,9 @@ XMem *diskBuf;
static TAILQ_HEAD(CacheHashTable, DiskCacheEntry) *hashTable;
static TAILQ_HEAD(LRUCacheList, DiskCacheEntry) lruList;
static uint64_t cacheHit;
static uint64_t cacheMiss;
static uint64_t cacheAlloc;
#define CACHESIZE (16*1024*1024)
#define HASHTABLEENTRIES 128
@ -55,6 +58,10 @@ DiskCache_Init()
e->buffer = (void *)(bufBase + BLOCKSIZE * i);
TAILQ_INSERT_TAIL(&lruList, e, lruEntry);
}
cacheHit = 0;
cacheMiss = 0;
cacheAlloc = 0;
}
int
@ -125,6 +132,8 @@ DiskCache_Alloc(Disk *disk, uint64_t diskOffset, DiskCacheEntry **entry)
status = DiskCacheAlloc(disk, diskOffset, entry);
}
cacheAlloc++;
Spinlock_Unlock(&cacheLock);
return status;
@ -153,9 +162,12 @@ DiskCache_Read(Disk *disk, uint64_t diskOffset, DiskCacheEntry **entry)
Spinlock_Lock(&cacheLock);
status = DiskCacheLookup(disk, diskOffset, entry);
if (*entry != NULL) {
cacheHit++;
Spinlock_Unlock(&cacheLock);
return status;
}
cacheMiss++;
status = DiskCacheAlloc(disk, diskOffset, entry);
if (status != 0) {
Spinlock_Unlock(&cacheLock);
@ -188,3 +200,13 @@ DiskCache_Write(DiskCacheEntry *entry)
return Disk_Write(entry->disk, buf, &sga, NULL, NULL);
}
void
Debug_DiskCache(int argc, const char *argv[])
{
kprintf("Hits: %lld\n", cacheHit);
kprintf("Misses: %lld\n", cacheMiss);
kprintf("Allocations: %lld\n", cacheAlloc);
}
REGISTER_DBGCMD(diskcache, "Display disk cache statistics", Debug_DiskCache);

101
sys/kern/vfs.c Normal file
View File

@ -0,0 +1,101 @@
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <sys/kassert.h>
#include <sys/kdebug.h>
#include <sys/spinlock.h>
#include <sys/disk.h>
#include <sys/vfs.h>
extern VFS *O2FS_Mount(Disk *root);
static Spinlock vfsLock;
static VFS *rootFS;
static VNode *rootNode;
int
VFS_MountRoot(Disk *rootDisk)
{
int status;
Spinlock_Init(&vfsLock, "VFS Lock");
rootFS = O2FS_Mount(rootDisk);
if (!rootFS)
return -1;
status = rootFS->op->getroot(rootFS, &rootNode);
if (status < 0)
Panic("Failed to get root VNode\n");
return 0;
}
VNode *
VFS_Lookup(const char *path)
{
int status;
const char *start = path + 1;
const char *end = path + 1;
uint64_t len;
VNode *curNode;
VNode *oldNode;
char curName[256];
if (path[0] != '/')
return NULL;
status = rootFS->op->getroot(rootFS, &curNode);
if (status < 0)
Panic("Failed to get root VNode\n");
while (1) {
while (*end != '\0' || *end != '/')
end++;
len = (size_t)(end - start);
if (len == 0 || len > 256) {
// Release
return NULL;
}
memcpy(curName, start, len);
curName[len - 1] = '\0';
oldNode = curNode;
status = curNode->op->lookup(curNode, &curNode, curName);
if (status < 0) {
// Release
return NULL;
}
// Release oldNode
if (*end == '\0')
return curNode;
start = end + 1;
end = end + 1;
}
}
int
VFS_Open(VNode *fn)
{
return fn->op->open(fn);
}
int
VFS_Close(VNode *fn)
{
return fn->op->close(fn);
}
int
VFS_Read(VNode *fn, void *buf, uint64_t off, uint64_t len)
{
return fn->op->read(fn, buf, off, len);
}