Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement block addressing format #16

Merged
merged 16 commits into from
Jun 20, 2024
125 changes: 103 additions & 22 deletions ext4/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,11 +181,80 @@ func (ext4 *FileSystem) listFileInfo(ino int64) ([]FileInfo, error) {
return fileInfos, nil
}

func extractDirectoryEntries(directoryReader *bytes.Buffer) ([]DirectoryEntry2, error) {
var dirEntries []DirectoryEntry2

for {
dirEntry := DirectoryEntry2{}

err := struc.Unpack(directoryReader, &dirEntry)
if err != nil {
if err == io.EOF {
break
}
return nil, xerrors.Errorf("failed to parse directory entry: %w", err)
}

if dirEntry.RecLen == 0 {
break
}

align := dirEntry.RecLen - uint16(dirEntry.NameLen+8)
_, err = directoryReader.Read(make([]byte, align))
if err != nil {
return nil, xerrors.Errorf("failed to read align: %w", err)
}

if dirEntry.Name == "." || dirEntry.Name == ".." {
continue
}
if dirEntry.Flags == 0xDE {
continue
}
if dirEntry.Flags == 0 {
continue
}

dirEntries = append(dirEntries, dirEntry)
}

return dirEntries, nil
}

func (ext4 *FileSystem) listEntries(ino int64) ([]DirectoryEntry2, error) {
inode, err := ext4.getInode(ino)
if err != nil {
return nil, xerrors.Errorf("failed to get root inode: %w", err)
}

if !inode.UsesExtents() {
var dirEntries []DirectoryEntry2

blockAddresses, err := inode.GetBlockAddresses(ext4)
if err != nil {
return nil, xerrors.Errorf("failed to get block address: %w", err)
}

for _, blockAddress := range blockAddresses {
_, err = ext4.r.Seek(int64(blockAddress)*ext4.sb.GetBlockSize(), 0)
if err != nil {
return nil, xerrors.Errorf("failed to seek: %w", err)
}

directoryReader, err := readBlock(ext4.r, ext4.sb.GetBlockSize())
if err != nil {
return nil, xerrors.Errorf("failed to read directory entry: %w", err)
}

extracted, err := extractDirectoryEntries(directoryReader)
if err != nil {
return nil, xerrors.Errorf("failed to extract directory entries: %w", err)
}
dirEntries = append(dirEntries, extracted...)
}
return dirEntries, nil
}

extents, err := ext4.Extents(inode)
if err != nil {
return nil, xerrors.Errorf("failed to get extents: %w", err)
Expand All @@ -202,28 +271,11 @@ func (ext4 *FileSystem) listEntries(ino int64) ([]DirectoryEntry2, error) {
return nil, xerrors.Errorf("failed to read directory entry: %w", err)
}

for {
dirEntry := DirectoryEntry2{}
err = struc.Unpack(directoryReader, &dirEntry)
if err != nil {
if err == io.EOF {
break
}
return nil, xerrors.Errorf("failed to parse directory entry: %w", err)
}
align := dirEntry.RecLen - uint16(dirEntry.NameLen+8)
_, err := directoryReader.Read(make([]byte, align))
if err != nil {
return nil, xerrors.Errorf("failed to read align: %w", err)
}
if dirEntry.Name == "." || dirEntry.Name == ".." {
continue
}
if dirEntry.Flags == 0xDE {
continue
}
entries = append(entries, dirEntry)
dirEntries, err := extractDirectoryEntries(directoryReader)
if err != nil {
return nil, xerrors.Errorf("failed to extract directory entries: %w", err)
}
entries = append(entries, dirEntries...)
}
return entries, nil
}
Expand Down Expand Up @@ -304,7 +356,12 @@ func (ext4 *FileSystem) Open(name string) (fs.File, error) {
inode: dir.inode,
mode: fs.FileMode(dir.inode.Mode),
}
f, err := ext4.file(fi, name)
var f *File
if fi.inode.UsesExtents() {
f, err = ext4.file(fi, name)
} else {
f, err = ext4.fileFromBlock(fi, name)
}
if err != nil {
return nil, xerrors.Errorf("failed to get file(inode: %d): %w", dir.ino, err)
}
Expand All @@ -313,6 +370,30 @@ func (ext4 *FileSystem) Open(name string) (fs.File, error) {
return nil, fs.ErrNotExist
}

func (ext4 *FileSystem) fileFromBlock(fi FileInfo, filePath string) (*File, error) {
blockAddresses, err := fi.inode.GetBlockAddresses(ext4)
if err != nil {
return nil, xerrors.Errorf("failed to get block addresses: %w", err)
}

dt := make(dataTable)
for i, blockAddress := range blockAddresses {
offset := int64(blockAddress) * ext4.sb.GetBlockSize()
dt[int64(i)] = offset
}

return &File{
fs: ext4,
FileInfo: fi,
currentBlock: -1,
buffer: bytes.NewBuffer(nil),
filePath: filePath,
blockSize: ext4.sb.GetBlockSize(),
table: dt,
size: fi.Size(),
}, nil
}

func (ext4 *FileSystem) file(fi FileInfo, filePath string) (*File, error) {
extents, err := ext4.extents(fi.inode.BlockOrExtents[:], nil)
if err != nil {
Expand Down
138 changes: 138 additions & 0 deletions ext4/inode.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
package ext4

import (
"bytes"
"encoding/binary"

"golang.org/x/xerrors"
)

// ExtentHeader is ...
type ExtentHeader struct {
Magic uint16 `struc:"uint16,little"`
Expand Down Expand Up @@ -66,6 +73,13 @@ type Inode struct {
Reserved [96]uint8 `struc:"[96]uint32,little"`
}

type BlockAddressing struct {
DirectBlock [12]uint32 `struc:"[12]uint32,little"`
SingleIndirectBlock uint32 `struc:"uint32,little"`
DoubleIndirectBlock uint32 `struc:"uint32,little"`
TripleIndirectBlock uint32 `struc:"uint32,little"`
}

func (i Inode) IsDir() bool {
return i.Mode&0x4000 != 0 && i.Mode&0x8000 == 0
}
Expand Down Expand Up @@ -97,6 +111,130 @@ func (i *Inode) GetSize() int64 {
return (int64(i.SizeHigh) << 32) | int64(i.SizeLo)
}

func resolveSingleIndirectBlockAddress(ext4 *FileSystem, singleIndirectBlockAddress uint32) ([]uint32, error) {
var blockAddresses []uint32

_, err := ext4.r.Seek(int64(singleIndirectBlockAddress)*ext4.sb.GetBlockSize(), 0)
if err != nil {
return nil, xerrors.Errorf("failed to seek: %w", err)
}

singleIndirectBlockAddresses, err := readBlock(ext4.r, ext4.sb.GetBlockSize())
if err != nil {
return nil, xerrors.Errorf("failed to read directory entry at block address %#x: %w", singleIndirectBlockAddress, err)
}

for singleIndirectBlockAddresses.Len() > 0 {
address := binary.LittleEndian.Uint32(singleIndirectBlockAddresses.Next(4))
if address == 0 {
break
}
blockAddresses = append(blockAddresses, address)
}

return blockAddresses, nil
}

func resolveDoubleIndirectBlockAddress(ext4 *FileSystem, doubleIndirectBlockAddress uint32) ([]uint32, error) {
var blockAddresses []uint32

_, err := ext4.r.Seek(int64(doubleIndirectBlockAddress)*ext4.sb.GetBlockSize(), 0)
if err != nil {
return nil, xerrors.Errorf("failed to seek: %w", err)
}

doubleIndirectBlockAddresses, err := readBlock(ext4.r, ext4.sb.GetBlockSize())
if err != nil {
return nil, xerrors.Errorf("failed to read directory entry at block address %#x: %w", doubleIndirectBlockAddress, err)
}

for doubleIndirectBlockAddresses.Len() > 0 {
singleIndirectBlockAddress := binary.LittleEndian.Uint32(doubleIndirectBlockAddresses.Next(4))
if singleIndirectBlockAddress == 0 {
break
}

singleIndirectBlockAddresses, err := resolveSingleIndirectBlockAddress(ext4, singleIndirectBlockAddress)
if err != nil {
return nil, xerrors.Errorf("failed to read single indirect block addressing: %w", err)
}
blockAddresses = append(blockAddresses, singleIndirectBlockAddresses...)
}

return blockAddresses, nil
}

func resolveTripleIndirectBlockAddress(ext4 *FileSystem, tripleIndirectBlockAddress uint32) ([]uint32, error) {
var blockAddresses []uint32

_, err := ext4.r.Seek(int64(tripleIndirectBlockAddress)*ext4.sb.GetBlockSize(), 0)
if err != nil {
return nil, xerrors.Errorf("failed to seek: %w", err)
}

tripleIndirectBlockAddresses, err := readBlock(ext4.r, ext4.sb.GetBlockSize())
if err != nil {
return nil, xerrors.Errorf("failed to read directory entry at block address %#x: %w", tripleIndirectBlockAddress, err)
}

for tripleIndirectBlockAddresses.Len() > 0 {
doubleIndirectBlockAddress := binary.LittleEndian.Uint32(tripleIndirectBlockAddresses.Next(4))
if doubleIndirectBlockAddress == 0 {
break
}

doubleIndirectBlockAddresses, err := resolveDoubleIndirectBlockAddress(ext4, doubleIndirectBlockAddress)
if err != nil {
return nil, xerrors.Errorf("failed to read double indirect block addressing: %w", err)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ここ全体的にですが、エラーが発生したアドレスがわかるようにしたいです。ブロックのアドレスなどdebugの時にxxd とかで参照できると嬉しい(他のところが十分にできてないので心苦しいですが)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ひとまず single/double/triple の block addresses を解決するところで、エラーメッセージ中にブロックアドレスを表示するようにしてみました。
22dd741

}
blockAddresses = append(blockAddresses, doubleIndirectBlockAddresses...)
}

return blockAddresses, nil
}

func (i *Inode) GetBlockAddresses(ext4 *FileSystem) ([]uint32, error) {
addresses := BlockAddressing{}
err := binary.Read(bytes.NewReader(i.BlockOrExtents[:]), binary.LittleEndian, &addresses)
if err != nil {
return nil, xerrors.Errorf("failed to read block addressing: %w", err)
}

var blockAddresses []uint32
for _, blockAddress := range addresses.DirectBlock {
if blockAddress == 0 {
break
}
blockAddresses = append(blockAddresses, blockAddress)
}

if addresses.SingleIndirectBlock != 0 {
singleIndirectBlockAddresses, err := resolveSingleIndirectBlockAddress(ext4, addresses.SingleIndirectBlock)
if err != nil {
return nil, xerrors.Errorf("failed to read single indirect block addressing: %w", err)
}
blockAddresses = append(blockAddresses, singleIndirectBlockAddresses...)
}

if addresses.DoubleIndirectBlock != 0 {
doubleIndirectBlockAddresses, err := resolveDoubleIndirectBlockAddress(ext4, addresses.DoubleIndirectBlock)
if err != nil {
return nil, xerrors.Errorf("failed to read double indirect block addressing: %w", err)
}
blockAddresses = append(blockAddresses, doubleIndirectBlockAddresses...)
}

if addresses.TripleIndirectBlock != 0 {
tripleIndirectBlockAddresses, err := resolveTripleIndirectBlockAddress(ext4, addresses.TripleIndirectBlock)
if err != nil {
return nil, xerrors.Errorf("failed to read triple indirect block addressing: %w", err)
}
blockAddresses = append(blockAddresses, tripleIndirectBlockAddresses...)
}
Comment on lines +211 to +233
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

これって全て存在するパターンや一部存在するパターンなどあるんですかね?
どういった組み合わせが存在するのか気になってます

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Block Addressingの制約で、1ファイルのサイズが1Blockのサイズを超えたときのためにこのような管理方法が取られているようです。

https://ext4.wiki.kernel.org/index.php/Ext4_Disk_Layout#Direct.2FIndirect_Block_Addressing

ここからサイズを計算すると、ファイルサイズが4GBを超えるとTripleまで全て存在するようになります。4MBを超えるとDoubleまで、4KBを超えるとSingleまでになるはずです。少ない方から順番に詰められていくので、途中で飛ばしたりという組み合わせはなさそうです。

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

なるほどです。拡張されていく仕様ですか。
ありがとうございます。


return blockAddresses, nil
}

// ExtentInternal
type ExtentInternal struct {
Block uint32 `struc:"uint32,little"`
Expand Down