HeroFS - Distributed Filesystem for HeroLib

HeroFS is a distributed filesystem implementation built on top of HeroDB (Redis-based storage). It provides a virtual filesystem with support for files, directories, symbolic links, and binary data blobs.

Overview

HeroFS implements a filesystem structure where:

• Fs: Represents a filesystem as a top-level container
• FsDir: Represents directories within a filesystem
• FsFile: Represents files with support for multiple directory associations
• FsSymlink: Represents symbolic links pointing to files or directories
• FsBlob: Represents binary data chunks (up to 1MB) used as file content

Features

• Distributed storage using Redis
• Support for files, directories, and symbolic links
• Blob-based file content storage with integrity verification
• Multiple directory associations for files (similar to hard links)
• Filesystem quotas and usage tracking
• Metadata support for files
• Efficient lookup mechanisms using Redis hash sets

Installation

HeroFS is part of HeroLib and is automatically available when using HeroLib.

Usage

To use HeroFS, you need to create a filesystem factory:

import incubaid.herolib.hero.herofs

mut fs_factory := herofs.new()!

Creating a Filesystem

mut fs := fs_factory.fs.new(
 name: 'my_filesystem'
 quota_bytes: 1000000000 // 1GB quota
)!
fs = fs_factory.fs.set(fs)!
fs_id := fs.id

Working with Directories

// Create root directory
mut root_dir := fs_factory.fs_dir.new(
 name: 'root'
 fs_id: fs_id
 parent_id: 0
)!
root_dir_id := fs_factory.fs_dir.set(root_dir)!

// Create subdirectory
mut sub_dir := fs_factory.fs_dir.new(
 name: 'documents'
 fs_id: fs_id
 parent_id: root_dir_id
)!
sub_dir_id := fs_factory.fs_dir.set(sub_dir)!

Working with Blobs

// Create a blob with binary data
mut blob := fs_factory.fs_blob.new(
 data: content_bytes
 mime_type: 'text/plain'
)!
blob_id := fs_factory.fs_blob.set(blob)!

Working with Files

// Create a file
mut file := fs_factory.fs_file.new(
 name: 'example.txt'
 fs_id: fs_id
 directories: [root_dir_id]
 blobs: [blob_id]
)!
file_id := fs_factory.fs_file.set(file)!

Working with Symbolic Links

// Create a symbolic link to a file
mut symlink := fs_factory.fs_symlink.new(
 name: 'example_link.txt'
 fs_id: fs_id
 parent_id: root_dir_id
 target_id: file_id
 target_type: .file
)!
symlink_id := fs_factory.fs_symlink.set(symlink)!

API Reference

The HeroFS module provides the following main components:

• FsFactory - Main factory for accessing all filesystem components
• DBFs - Filesystem operations
• DBFsDir - Directory operations
• DBFsFile - File operations
• DBFsSymlink - Symbolic link operations
• DBFsBlob - Binary data blob operations

Each component provides CRUD operations and specialized methods for filesystem management.

Examples

Check the examples/hero/herofs/ directory for detailed usage examples.

fn delete_fs_test() !

fn extension_to_mime_type(ext string) MimeType

extension_to_mime_type converts file extension to MimeType enum

fn fs_blob_handle(mut f FSFactory, rpcid int, servercontext map[string]string, userref UserRef, method string, params string) !Response

fn fs_blob_membership_handle(mut f FSFactory, rpcid int, servercontext map[string]string, userref UserRef, method string, params string) !Response

fn fs_dir_handle(mut f FSFactory, rpcid int, servercontext map[string]string, userref UserRef, method string, params string) !Response

fn fs_file_handle(mut f FSFactory, rpcid int, servercontext map[string]string, userref UserRef, method string, params string) !Response

fn fs_handle(mut f FSFactory, rpcid int, servercontext map[string]string, userref UserRef, method string, params string) !Response

fn fs_symlink_handle(mut f FSFactory, rpcid int, servercontext map[string]string, userref UserRef, method string, params string) !Response

fn mime_type_to_string(m MimeType) string

fn new(args DBArgs) !FSFactory

fn new_fs(args FsArg) !Fs

Convenience function for creating a filesystem

fn new_fs_test() !Fs

fn new_test() !FSFactory

fn string_to_mime_type(s string) ?MimeType

enum FSItemType {
	file
	directory
	symlink
}

FSItemType indicates what type of filesystem object was found

enum MimeType {
	aac
	abiword
	apng
	freearc
	avif
	avi
	azw
	bin
	bmp
	bz
	bz2
	cda
	csh
	css
	csv
	doc
	docx
	eot
	epub
	gz
	gif
	html
	ico
	ics
	jar
	jpg
	js
	json
	jsonld
	md
	midi
	mjs
	mp3
	mp4
	mpeg
	mpkg
	odp
	ods
	odt
	oga
	ogv
	ogx
	opus
	otf
	png
	pdf
	php
	ppt
	pptx
	rar
	rtf
	sh
	svg
	tar
	tiff
	ts
	ttf
	txt
	vsd
	wav
	weba
	webm
	manifest
	webp
	woff
	woff2
	xhtml
	xls
	xlsx
	xml
	xul
	zip
	gp3
	gpp2
	sevenz
}

see https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types/Common_types

enum SymlinkTargetType {
	file
	directory
}

struct BlobList {
pub mut:
	id   u32
	hash string
	size int
}

BlobList represents a simplified blob structure for listing purposes

struct CopyOptions {
pub mut:
	recursive  bool = true // Copy directories recursively
	overwrite  bool // Overwrite existing files at destination
	copy_blobs bool // Create new blob copies (true) or reference same blobs (false)
}

CopyOptions provides options for copy operations

struct DBArgs {
pub mut:
	redis ?&redisclient.Redis
}

struct DBFs {
pub mut:
	db      &db.DB     @[skip; str: skip]
	factory &FSFactory = unsafe { nil } @[skip; str: skip]
}

We only keep the root directory ID here, other directories can be found by querying parent_id in FsDir

fn (mut self DBFs) new(args FsArg) !Fs

get new filesystem, not from the DB

fn (mut self DBFs) new_get_set(args_ FsArg) !Fs

get new filesystem, if it exists then it will get it from the DB

fn (mut self DBFs) set(o Fs) !Fs

fn (mut self DBFs) delete(id u32) !

fn (mut self DBFs) exist(id u32) !bool

fn (mut self DBFs) get(id u32) !Fs

fn (mut self DBFs) list(args FsListArg) ![]Fs

fn (mut self DBFs) get_by_name(name string) !Fs

Additional hset operations for efficient lookups

fn (mut self DBFs) increase_usage(id u32, bytes u64) !

Increase used bytes counter

fn (mut self DBFs) decrease_usage(id u32, bytes u64) !

Decrease used bytes counter

fn (mut self DBFs) check_quota(id u32, additional_bytes u64) !bool

Check if quota is exceeded

struct DBFsBlob {
pub mut:
	db      &db.DB     @[skip; str: skip]
	factory &FSFactory = unsafe { nil } @[skip; str: skip]
}

Update DBFsBlob struct:

fn (mut self DBFsBlob) new(args FsBlobArg) !FsBlob

get new blob, not from the DB

fn (mut self DBFsBlob) set(o_ FsBlob) !FsBlob

fn (mut self DBFsBlob) delete(id u32) !

fn (mut self DBFsBlob) delete_multi(ids []u32) !

fn (mut self DBFsBlob) exist(id u32) !bool

fn (mut self DBFsBlob) exist_multi(ids []u32) !bool

fn (mut self DBFsBlob) get(id u32) !FsBlob

fn (mut self DBFsBlob) get_multi(id []u32) ![]FsBlob

fn (mut self DBFsBlob) list() ![]FsBlob

fn (mut self DBFsBlob) get_by_hash(hash string) !FsBlob

fn (mut self DBFsBlob) exists_by_hash(hash string) !bool

fn (mut self DBFsBlob) verify(hash string) !bool

struct DBFsBlobMembership {
pub mut:
	db      &db.DB     @[skip; str: skip]
	factory &FSFactory = unsafe { nil } @[skip; str: skip]
}

fn (self DBFsBlobMembership) new(args FsBlobMembershipArg) !FsBlobMembership

fn (mut self DBFsBlobMembership) set(o FsBlobMembership) !FsBlobMembership

fn (mut self DBFsBlobMembership) delete(hash string) !

fn (mut self DBFsBlobMembership) exist(hash string) !bool

fn (mut self DBFsBlobMembership) get(hash string) !FsBlobMembership

fn (mut self DBFsBlobMembership) add_filesystem(hash string, fs_id u32) !

Add a filesystem to an existing blob membership

fn (mut self DBFsBlobMembership) remove_filesystem(hash string, fs_id u32) !

Remove a filesystem from an existing blob membership

fn (mut self DBFsBlobMembership) list_prefix(prefix string) ![]FsBlobMembership

list_by_hash_prefix lists blob memberships where hash starts with the given prefix Returns maximum 10000 items as FsBlobMembership entries

struct DBFsDir {
pub mut:
	db      &db.DB     @[skip; str: skip]
	factory &FSFactory = unsafe { nil } @[skip; str: skip]
}

fn (mut self DBFsDir) new(args FsDirArg) !FsDir

get new directory, not from the DB

fn (mut self DBFsDir) set(o FsDir) !FsDir

fn (mut self DBFsDir) delete(id u32) !

fn (mut self DBFsDir) exist(id u32) !bool

fn (mut self DBFsDir) get(id u32) !FsDir

fn (mut self DBFsDir) create_path(fs_id u32, path string) !u32

create_path creates a directory at the specified path, creating parent directories as needed

fn (mut self DBFsDir) list() ![]FsDir

List all directories

fn (mut self DBFsDir) list_by_filesystem(fs_id u32) ![]FsDir

List directories in a filesystem

fn (mut self DBFsDir) list_children(dir_id u32) ![]FsDir

List child directories

fn (mut self DBFsDir) has_children(dir_id u32) !bool

Check if directory has children

fn (mut self DBFsDir) rename(id u32, new_name string) !

Rename directory

fn (mut self DBFsDir) move(id u32, new_parent_id u32) !

Move directory to a new parent

struct DBFsFile {
pub mut:
	db      &db.DB     @[skip; str: skip]
	factory &FSFactory = unsafe { nil } @[skip; str: skip]
}

fn (mut self DBFsFile) new(args FsFileArg) !FsFile

get new file, not from the DB

fn (mut self DBFsFile) set(o_ FsFile) !FsFile

fn (mut self DBFsFile) add_to_directory(file_id u32, dir_id u32) !

add_to_directory adds a file to a directory's files list

fn (mut self DBFsFile) remove_from_directory(file_id u32, dir_id u32) !

remove_from_directory removes a file from a directory's files list

fn (mut self DBFsFile) delete(id u32) !

fn (mut self DBFsFile) exist(id u32) !bool

fn (mut self DBFsFile) get(id u32) !FsFile

fn (mut self DBFsFile) update_accessed(id u32) !

Update file accessed timestamp

fn (mut self DBFsFile) update_metadata(id u32, key string, value string) !

Update file metadata

fn (mut self DBFsFile) rename(id u32, new_name string) !

Rename file (affects all directories)

fn (mut self DBFsFile) move(id u32, new_dir_ids []u32) !

Move file to different directories

fn (mut self DBFsFile) append_blob(id u32, blob_id u32) !

Append a blob to the file

fn (mut self DBFsFile) list() ![]FsFile

List all files

fn (mut self DBFsFile) get_by_path(dir_id u32, name string) !FsFile

Get file by path (directory and name)

fn (mut self DBFsFile) list_by_directory(dir_id u32) ![]FsFile

List files in a directory

fn (mut self DBFsFile) list_by_filesystem(fs_id u32) ![]FsFile

List files in a filesystem

fn (mut self DBFsFile) list_by_mime_type(mime_type MimeType) ![]FsFile

List files by MIME type

fn (mut self DBFsFile) list_directories_for_file(file_id u32) ![]u32

Helper method to find which directories contain a file

struct DBFsSymlink {
pub mut:
	db      &db.DB     @[skip; str: skip]
	factory &FSFactory = unsafe { nil } @[skip; str: skip]
}

fn (mut self DBFsSymlink) new(args FsSymlinkArg) !FsSymlink

get new symlink, not from the DB

fn (mut self DBFsSymlink) set(o FsSymlink) !FsSymlink

fn (mut self DBFsSymlink) delete(id u32) !

fn (mut self DBFsSymlink) exist(id u32) !bool

fn (mut self DBFsSymlink) get(id u32) !FsSymlink

fn (mut self DBFsSymlink) list() ![]FsSymlink

List all symlinks

fn (mut self DBFsSymlink) list_by_filesystem(fs_id u32) ![]FsSymlink

List symlinks in a filesystem

fn (mut self DBFsSymlink) is_broken(id u32) !bool

Check if symlink is broken (target doesn't exist)

struct ExportOptions {
pub mut:
	recursive     bool = true // Export directories recursively
	overwrite     bool // Overwrite existing files on real filesystem
	preserve_meta bool = true // Preserve file metadata (timestamps, etc.)
}

ExportOptions provides options for export operations

struct FSFactory {
pub mut:
	fs                 DBFs
	fs_blob            DBFsBlob
	fs_blob_membership DBFsBlobMembership
	fs_dir             DBFsDir
	fs_file            DBFsFile
	fs_symlink         DBFsSymlink
}

struct FindOptions {
pub mut:
	recursive        bool = true
	include_patterns []string // File/directory name patterns to include (e.g. ['*.v', 'doc*'])
	exclude_patterns []string // File/directory name patterns to exclude
	max_depth        int = -1 // Maximum depth to search (-1 for unlimited)
	follow_symlinks  bool // Whether to follow symbolic links during search
}

FindOptions provides options for filesystem search operations

struct FindResult {
pub mut:
	result_type FSItemType
	id          u32
	path        string
}

FindResult represents the result of a filesystem search

struct Fs {
	db.Base
pub mut:
	name        string
	group_id    u32 // Associated group for permissions
	root_dir_id u32 // ID of root directory
	quota_bytes u64 // Storage quota in bytes
	used_bytes  u64 // Current usage in bytes
	factory     &FSFactory = unsafe { nil } @[skip; str: skip]
}

Fs represents a filesystem, is the top level container for files and directories and symlinks, blobs are used over filesystems

fn (mut self Fs) cp(src_path string, dest_path string, find_opts FindOptions, copy_opts CopyOptions) !

cp copies files and directories from source path to destination

Parameters:- src_path: Source path pattern (can use wildcards via FindOptions)

• dest_path: Destination path
• find_opts: FindOptions for filtering source items
• copy_opts: CopyOptions for copy behavior

fs.cp('/src/*.v', '/backup/', FindOptions{recursive: true}, CopyOptions{overwrite: true})!

fn (self Fs) description(methodname string) string

return example rpc call and result for each methodname

fn (self Fs) dump(mut e encoder.Encoder) !

fn (self Fs) example(methodname string) (string, string)

return example rpc call and result for each methodname

fn (mut self Fs) export(src string, dest string, opts ExportOptions) !

export copies data from VFS to the real filesystem

Parameters:- src: Source path in VFS

• dest: Destination path on real filesystem
• opts: ExportOptions for export behavior

fs.export('/documents', '/home/user/backup', ExportOptions{recursive: true, overwrite: true})!

fn (mut self Fs) find(start_path string, opts FindOptions) ![]FindResult

find searches for filesystem objects starting from a given path

Parameters:- start_path: The path to start searching from

• opts: FindOptions struct with search parameters

Returns:- []FindResult: Array of found filesystem objects

results := tools.find('/', FindOptions{
    recursive: true
    include_patterns: ['*.v']
    exclude_patterns: ['*test*']
})!

fn (mut self Fs) get_abs_path_for_item(id u32, item_type FSItemType) !string

get_abs_path_for_item returns the absolute path for a given filesystem item ID and type

fn (mut self Fs) get_dir_by_absolute_path(path string) !FsDir

get_dir_by_absolute_path resolves an absolute path to a directory ID

Parameters:- path: The absolute path to resolve (e.g., "/", "/home", "/home/user/documents")

Returns:- FsDir: The directory object at the specified path

dir := tools.get_dir_by_absolute_path('/home/user/documents')!

fn (mut self Fs) get_dir_path(dir_id u32) !string

get_dir_path returns the absolute path for a given directory ID.

fn (mut self Fs) get_file_by_absolute_path(path string) !FsFile

get_file_by_absolute_path resolves an absolute path to a file

Parameters:- path: The absolute path to resolve (e.g., "/home/user/document.txt")

Returns:- FsFile: The file object at the specified path

file := tools.get_file_by_absolute_path('/home/user/document.txt')!

fn (mut self Fs) get_symlink_by_absolute_path(path string) !FsSymlink

get_symlink_by_absolute_path resolves an absolute path to a symlink

Parameters:- path: The absolute path to resolve (e.g., "/home/user/link.txt")

Returns:- FsSymlink: The symlink object at the specified path

symlink := tools.get_symlink_by_absolute_path('/home/user/link.txt')!

fn (mut self Fs) import(src string, dest string, opts ImportOptions) !

import copies data from the real filesystem into the VFS

Parameters:- src: Source path on real filesystem

• dest: Destination path in VFS
• opts: ImportOptions for import behavior

fs.import('/home/user/documents', '/imported', ImportOptions{recursive: true, overwrite: true})!

fn (mut self Fs) mv(src_path string, dest_path string, opts MoveOptions) !

mv moves files and directories from source path to destination

Parameters:- src_path: Source path (exact path, not pattern)

• dest_path: Destination path
• opts: MoveOptions for move behavior

fs.mv('/src/main.v', '/backup/main.v', MoveOptions{overwrite: true})!

fn (mut self Fs) rm(path string, find_opts FindOptions, remove_opts RemoveOptions) !

rm removes files and directories matching the given path pattern

Parameters:- path: Path pattern to match for removal

• find_opts: FindOptions for filtering items to remove
• remove_opts: RemoveOptions for removal behavior

fs.rm('/temp/*', FindOptions{recursive: true}, RemoveOptions{recursive: true, delete_blobs: true})!

fn (mut self Fs) root_dir() !FsDir

fn (self Fs) type_name() string

struct FsArg {
pub mut:
	name        string @[required]
	description string
	group_id    u32
	root_dir_id u32
	quota_bytes u64
	used_bytes  u64
	tags        []string
	messages    []db.MessageArg
}

struct FsBlob {
	db.Base
pub mut:
	hash       string // blake192 hash of content
	data       []u8   // Binary data (max 1MB)
	size_bytes int    // Size in bytes
	created_at i64
	mime_type  string // MIME type
	encoding   string // Encoding type
}

FsBlob represents binary data up to 1MB

fn (self FsBlob) type_name() string

fn (self FsBlob) description(methodname string) string

return example rpc call and result for each methodname

fn (self FsBlob) example(methodname string) (string, string)

return example rpc call and result for each methodname

fn (self FsBlob) dump(mut e encoder.Encoder) !

fn (mut blob FsBlob) calculate_hash()

fn (blob FsBlob) verify_integrity() bool

struct FsBlobArg {
pub mut:
	data       []u8 @[required]
	mime_type  string
	encoding   string
	created_at i64
}

struct FsBlobMembership {
pub mut:
	hash   string // blake192 hash of content (key)
	fsid   []u32  // list of fs ids where this blob is used
	blobid u32    // id of the blob
}

FsBlobMembership represents membership of a blob in one or more filesystems, the key is the hash of the blob

fn (self FsBlobMembership) type_name() string

fn (self FsBlobMembership) dump(mut e encoder.Encoder) !

fn (self FsBlobMembership) description(methodname string) string

fn (self FsBlobMembership) example(methodname string) (string, string)

struct FsBlobMembershipArg {
pub mut:
	hash   string @[required]
	fsid   []u32  @[required]
	blobid u32    @[required]
}

struct FsDir {
	db.Base
pub mut:
	fs_id       u32 // Associated filesystem
	parent_id   u32 // Parent directory ID (0 for root)
	directories []u32
	files       []u32
	symlinks    []u32
}

FsDir represents a directory in a filesystem

fn (self FsDir) type_name() string

fn (self FsDir) dump(mut e encoder.Encoder) !

fn (self FsDir) description(methodname string) string

fn (self FsDir) example(methodname string) (string, string)

struct FsDirArg {
pub mut:
	name        string @[required]
	description string
	fs_id       u32 @[required]
	parent_id   u32
	tags        []string
	messages    []db.MessageArg
	directories []u32
	files       []u32
	symlinks    []u32
}

struct FsFile {
	db.Base
pub mut:
	fs_id       u32   // Associated filesystem
	directories []u32 // Directory IDs where this file exists
	blobs       []u32 // IDs of file content blobs
	size_bytes  u64
	mime_type   MimeType
	checksum    string // e.g., checksum of the file, needs to be calculated is blake 192
	accessed_at i64
	metadata    map[string]string // Custom metadata
}

FsFile represents a file in a filesystem

fn (self FsFile) type_name() string

fn (self FsFile) dump(mut e encoder.Encoder) !

fn (self FsFile) description(methodname string) string

fn (self FsFile) example(methodname string) (string, string)

struct FsFileArg {
pub mut:
	name        string @[required]
	description string
	fs_id       u32 @[required]
	directories []u32
	blobs       []u32
	size_bytes  u64
	mime_type   MimeType // Changed from string to MimeType enum
	checksum    string
	accessed_at i64
	metadata    map[string]string
	tags        []string
	messages    []db.MessageArg
}

struct FsListArg {
pub mut:
	group_id u32
	limit    int = 100 // Default limit is 100
}

struct FsSymlink {
	db.Base
pub mut:
	fs_id       u32 // Associated filesystem
	parent_id   u32 // Parent directory ID
	target_id   u32 // ID of target file or directory
	target_type SymlinkTargetType
}

FsSymlink represents a symbolic link in a filesystem

fn (self FsSymlink) type_name() string

fn (self FsSymlink) dump(mut e encoder.Encoder) !

fn (self FsSymlink) description(methodname string) string

fn (self FsSymlink) example(methodname string) (string, string)

struct FsSymlinkArg {
pub mut:
	name        string @[required]
	description string
	fs_id       u32               @[required]
	parent_id   u32               @[required]
	target_id   u32               @[required]
	target_type SymlinkTargetType @[required]
	tags        []string
	messages    []db.MessageArg
}

struct ImportOptions {
pub mut:
	recursive     bool = true // Import directories recursively
	overwrite     bool // Overwrite existing files in VFS
	preserve_meta bool = true // Preserve file metadata (timestamps, etc.)
}

ImportOptions provides options for import operations

struct MoveOptions {
pub mut:
	overwrite bool // Overwrite existing files at destination
}

MoveOptions provides options for move operations

struct RemoveOptions {
pub mut:
	recursive    bool // Remove directories recursively
	delete_blobs bool // Delete associated blobs (true) or keep them (false)
	force        bool // Force removal even if directory is not empty
}

RemoveOptions provides options for remove operations