From d5aee659f217746395ff58adf3a863627ff02ec1 Mon Sep 17 00:00:00 2001 From: Stephen Warren Date: Wed, 30 Jan 2019 12:58:05 -0700 Subject: [PATCH 01/16] fs: ext4: cache extent data When a file contains extents, U-Boot currently reads extent-related data for each block in the file, even if that data is located in the same block each time. This significantly slows down loading of files that use extents. Implement a very dumb cache to prevent repeatedly reading the same block. Files with extents now load as fast as files without. Note: There are many cases where read_allocated_block() is called. This patch only addresses one of those places; all others still read redundant data in any case they did before. This is a minimal patch to fix the load command; other cases aren't fixed. Signed-off-by: Stephen Warren --- fs/ext4/ext4_common.c | 45 ++++++++++++++++++++++--------------- fs/ext4/ext4_journal.c | 22 +++++++++--------- fs/ext4/ext4_write.c | 6 ++--- fs/ext4/ext4fs.c | 51 +++++++++++++++++++++++++++++++++++++----- include/ext4fs.h | 12 +++++++++- 5 files changed, 99 insertions(+), 37 deletions(-) diff --git a/fs/ext4/ext4_common.c b/fs/ext4/ext4_common.c index 67e2471bd3..29308e3b44 100644 --- a/fs/ext4/ext4_common.c +++ b/fs/ext4/ext4_common.c @@ -510,7 +510,8 @@ restart: restart_read: /* read the block no allocated to a file */ - first_block_no_of_root = read_allocated_block(g_parent_inode, blk_idx); + first_block_no_of_root = read_allocated_block(g_parent_inode, blk_idx, + NULL); if (first_block_no_of_root <= 0) goto fail; @@ -646,7 +647,7 @@ static int search_dir(struct ext2_inode *parent_inode, char *dirname) /* get the block no allocated to a file */ for (blk_idx = 0; blk_idx < directory_blocks; blk_idx++) { - blknr = read_allocated_block(parent_inode, blk_idx); + blknr = read_allocated_block(parent_inode, blk_idx, NULL); if (blknr <= 0) goto fail; @@ -943,7 +944,7 @@ int ext4fs_filename_unlink(char *filename) /* read the block no allocated to a file */ for (blk_idx = 0; blk_idx < directory_blocks; blk_idx++) { - blknr = read_allocated_block(g_parent_inode, blk_idx); + blknr = read_allocated_block(g_parent_inode, blk_idx, NULL); if (blknr <= 0) break; inodeno = unlink_filename(filename, blknr); @@ -1522,7 +1523,7 @@ void ext4fs_allocate_blocks(struct ext2_inode *file_inode, #endif static struct ext4_extent_header *ext4fs_get_extent_block - (struct ext2_data *data, char *buf, + (struct ext2_data *data, struct ext_block_cache *cache, struct ext4_extent_header *ext_block, uint32_t fileblock, int log2_blksz) { @@ -1551,12 +1552,10 @@ static struct ext4_extent_header *ext4fs_get_extent_block block = le16_to_cpu(index[i].ei_leaf_hi); block = (block << 32) + le32_to_cpu(index[i].ei_leaf_lo); - - if (ext4fs_devread((lbaint_t)block << log2_blksz, 0, blksz, - buf)) - ext_block = (struct ext4_extent_header *)buf; - else + block <<= log2_blksz; + if (!ext_cache_read(cache, (lbaint_t)block, blksz)) return NULL; + ext_block = (struct ext4_extent_header *)cache->buf; } } @@ -1613,7 +1612,8 @@ int ext4fs_read_inode(struct ext2_data *data, int ino, struct ext2_inode *inode) return 1; } -long int read_allocated_block(struct ext2_inode *inode, int fileblock) +long int read_allocated_block(struct ext2_inode *inode, int fileblock, + struct ext_block_cache *cache) { long int blknr; int blksz; @@ -1630,20 +1630,26 @@ long int read_allocated_block(struct ext2_inode *inode, int fileblock) if (le32_to_cpu(inode->flags) & EXT4_EXTENTS_FL) { long int startblock, endblock; - char *buf = zalloc(blksz); - if (!buf) - return -ENOMEM; + struct ext_block_cache *c, cd; struct ext4_extent_header *ext_block; struct ext4_extent *extent; int i; + + if (cache) { + c = cache; + } else { + c = &cd; + ext_cache_init(c); + } ext_block = - ext4fs_get_extent_block(ext4fs_root, buf, + ext4fs_get_extent_block(ext4fs_root, c, (struct ext4_extent_header *) inode->b.blocks.dir_blocks, fileblock, log2_blksz); if (!ext_block) { printf("invalid extent block\n"); - free(buf); + if (!cache) + ext_cache_fini(c); return -EINVAL; } @@ -1655,19 +1661,22 @@ long int read_allocated_block(struct ext2_inode *inode, int fileblock) if (startblock > fileblock) { /* Sparse file */ - free(buf); + if (!cache) + ext_cache_fini(c); return 0; } else if (fileblock < endblock) { start = le16_to_cpu(extent[i].ee_start_hi); start = (start << 32) + le32_to_cpu(extent[i].ee_start_lo); - free(buf); + if (!cache) + ext_cache_fini(c); return (fileblock - startblock) + start; } } - free(buf); + if (!cache) + ext_cache_fini(c); return 0; } diff --git a/fs/ext4/ext4_journal.c b/fs/ext4/ext4_journal.c index 148593da7f..6adbab93a6 100644 --- a/fs/ext4/ext4_journal.c +++ b/fs/ext4/ext4_journal.c @@ -347,7 +347,7 @@ void recover_transaction(int prev_desc_logical_no) ext4fs_read_inode(ext4fs_root, EXT2_JOURNAL_INO, (struct ext2_inode *)&inode_journal); blknr = read_allocated_block((struct ext2_inode *) - &inode_journal, i); + &inode_journal, i, NULL); ext4fs_devread((lbaint_t)blknr * fs->sect_perblk, 0, fs->blksz, temp_buff); p_jdb = (char *)temp_buff; @@ -372,7 +372,7 @@ void recover_transaction(int prev_desc_logical_no) be32_to_cpu(jdb->h_sequence)) == 0) continue; } - blknr = read_allocated_block(&inode_journal, i); + blknr = read_allocated_block(&inode_journal, i, NULL); ext4fs_devread((lbaint_t)blknr * fs->sect_perblk, 0, fs->blksz, metadata_buff); put_ext4((uint64_t)((uint64_t)be32_to_cpu(tag->block) * (uint64_t)fs->blksz), @@ -419,7 +419,8 @@ int ext4fs_check_journal_state(int recovery_flag) } ext4fs_read_inode(ext4fs_root, EXT2_JOURNAL_INO, &inode_journal); - blknr = read_allocated_block(&inode_journal, EXT2_JOURNAL_SUPERBLOCK); + blknr = read_allocated_block(&inode_journal, EXT2_JOURNAL_SUPERBLOCK, + NULL); ext4fs_devread((lbaint_t)blknr * fs->sect_perblk, 0, fs->blksz, temp_buff); jsb = (struct journal_superblock_t *) temp_buff; @@ -443,7 +444,7 @@ int ext4fs_check_journal_state(int recovery_flag) i = be32_to_cpu(jsb->s_first); while (1) { - blknr = read_allocated_block(&inode_journal, i); + blknr = read_allocated_block(&inode_journal, i, NULL); memset(temp_buff1, '\0', fs->blksz); ext4fs_devread((lbaint_t)blknr * fs->sect_perblk, 0, fs->blksz, temp_buff1); @@ -537,7 +538,7 @@ end: ext4_read_superblock((char *)fs->sb); blknr = read_allocated_block(&inode_journal, - EXT2_JOURNAL_SUPERBLOCK); + EXT2_JOURNAL_SUPERBLOCK, NULL); put_ext4((uint64_t) ((uint64_t)blknr * (uint64_t)fs->blksz), (struct journal_superblock_t *)temp_buff, (uint32_t) fs->blksz); @@ -566,7 +567,7 @@ static void update_descriptor_block(long int blknr) ext4fs_read_inode(ext4fs_root, EXT2_JOURNAL_INO, &inode_journal); jsb_blknr = read_allocated_block(&inode_journal, - EXT2_JOURNAL_SUPERBLOCK); + EXT2_JOURNAL_SUPERBLOCK, NULL); ext4fs_devread((lbaint_t)jsb_blknr * fs->sect_perblk, 0, fs->blksz, temp_buff); jsb = (struct journal_superblock_t *) temp_buff; @@ -618,7 +619,7 @@ static void update_commit_block(long int blknr) ext4fs_read_inode(ext4fs_root, EXT2_JOURNAL_INO, &inode_journal); jsb_blknr = read_allocated_block(&inode_journal, - EXT2_JOURNAL_SUPERBLOCK); + EXT2_JOURNAL_SUPERBLOCK, NULL); ext4fs_devread((lbaint_t)jsb_blknr * fs->sect_perblk, 0, fs->blksz, temp_buff); jsb = (struct journal_superblock_t *) temp_buff; @@ -645,16 +646,17 @@ void ext4fs_update_journal(void) long int blknr; int i; ext4fs_read_inode(ext4fs_root, EXT2_JOURNAL_INO, &inode_journal); - blknr = read_allocated_block(&inode_journal, jrnl_blk_idx++); + blknr = read_allocated_block(&inode_journal, jrnl_blk_idx++, NULL); update_descriptor_block(blknr); for (i = 0; i < MAX_JOURNAL_ENTRIES; i++) { if (journal_ptr[i]->blknr == -1) break; - blknr = read_allocated_block(&inode_journal, jrnl_blk_idx++); + blknr = read_allocated_block(&inode_journal, jrnl_blk_idx++, + NULL); put_ext4((uint64_t) ((uint64_t)blknr * (uint64_t)fs->blksz), journal_ptr[i]->buf, fs->blksz); } - blknr = read_allocated_block(&inode_journal, jrnl_blk_idx++); + blknr = read_allocated_block(&inode_journal, jrnl_blk_idx++, NULL); update_commit_block(blknr); printf("update journal finished\n"); } diff --git a/fs/ext4/ext4_write.c b/fs/ext4/ext4_write.c index 4eb77c327e..95ffa3dfad 100644 --- a/fs/ext4/ext4_write.c +++ b/fs/ext4/ext4_write.c @@ -479,7 +479,7 @@ static int ext4fs_delete_file(int inodeno) /* release data blocks */ for (i = 0; i < no_blocks; i++) { - blknr = read_allocated_block(&inode, i); + blknr = read_allocated_block(&inode, i, NULL); if (blknr == 0) continue; if (blknr < 0) @@ -695,7 +695,7 @@ void ext4fs_deinit(void) ext4fs_read_inode(ext4fs_root, EXT2_JOURNAL_INO, &inode_journal); blknr = read_allocated_block(&inode_journal, - EXT2_JOURNAL_SUPERBLOCK); + EXT2_JOURNAL_SUPERBLOCK, NULL); ext4fs_devread((lbaint_t)blknr * fs->sect_perblk, 0, fs->blksz, temp_buff); jsb = (struct journal_superblock_t *)temp_buff; @@ -776,7 +776,7 @@ static int ext4fs_write_file(struct ext2_inode *file_inode, long int blknr; int blockend = fs->blksz; int skipfirst = 0; - blknr = read_allocated_block(file_inode, i); + blknr = read_allocated_block(file_inode, i, NULL); if (blknr <= 0) return -1; diff --git a/fs/ext4/ext4fs.c b/fs/ext4/ext4fs.c index 2a28031d14..26db677a1f 100644 --- a/fs/ext4/ext4fs.c +++ b/fs/ext4/ext4fs.c @@ -62,6 +62,9 @@ int ext4fs_read_file(struct ext2fs_node *node, loff_t pos, lbaint_t delayed_next = 0; char *delayed_buf = NULL; short status; + struct ext_block_cache cache; + + ext_cache_init(&cache); if (blocksize <= 0) return -1; @@ -77,9 +80,11 @@ int ext4fs_read_file(struct ext2fs_node *node, loff_t pos, int blockoff = pos - (blocksize * i); int blockend = blocksize; int skipfirst = 0; - blknr = read_allocated_block(&(node->inode), i); - if (blknr < 0) + blknr = read_allocated_block(&node->inode, i, &cache); + if (blknr < 0) { + ext_cache_fini(&cache); return -1; + } blknr = blknr << log2_fs_blocksize; @@ -109,8 +114,10 @@ int ext4fs_read_file(struct ext2fs_node *node, loff_t pos, delayed_skipfirst, delayed_extent, delayed_buf); - if (status == 0) + if (status == 0) { + ext_cache_fini(&cache); return -1; + } previous_block_number = blknr; delayed_start = blknr; delayed_extent = blockend; @@ -136,8 +143,10 @@ int ext4fs_read_file(struct ext2fs_node *node, loff_t pos, delayed_skipfirst, delayed_extent, delayed_buf); - if (status == 0) + if (status == 0) { + ext_cache_fini(&cache); return -1; + } previous_block_number = -1; } /* Zero no more than `len' bytes. */ @@ -153,12 +162,15 @@ int ext4fs_read_file(struct ext2fs_node *node, loff_t pos, status = ext4fs_devread(delayed_start, delayed_skipfirst, delayed_extent, delayed_buf); - if (status == 0) + if (status == 0) { + ext_cache_fini(&cache); return -1; + } previous_block_number = -1; } *actread = len; + ext_cache_fini(&cache); return 0; } @@ -252,3 +264,32 @@ int ext4fs_uuid(char *uuid_str) return -ENOSYS; #endif } + +void ext_cache_init(struct ext_block_cache *cache) +{ + memset(cache, 0, sizeof(*cache)); +} + +void ext_cache_fini(struct ext_block_cache *cache) +{ + free(cache->buf); + ext_cache_init(cache); +} + +int ext_cache_read(struct ext_block_cache *cache, lbaint_t block, int size) +{ + /* This could be more lenient, but this is simple and enough for now */ + if (cache->buf && cache->block == block && cache->size == size) + return 1; + ext_cache_fini(cache); + cache->buf = malloc(size); + if (!cache->buf) + return 0; + if (!ext4fs_devread(block, 0, size, cache->buf)) { + free(cache->buf); + return 0; + } + cache->block = block; + cache->size = size; + return 1; +} diff --git a/include/ext4fs.h b/include/ext4fs.h index 2421011341..4b5de6e7b6 100644 --- a/include/ext4fs.h +++ b/include/ext4fs.h @@ -117,6 +117,12 @@ struct ext_filesystem { struct blk_desc *dev_desc; }; +struct ext_block_cache { + char *buf; + lbaint_t block; + int size; +}; + extern struct ext2_data *ext4fs_root; extern struct ext2fs_node *ext4fs_file; @@ -146,11 +152,15 @@ int ext4fs_size(const char *filename, loff_t *size); void ext4fs_free_node(struct ext2fs_node *node, struct ext2fs_node *currroot); int ext4fs_devread(lbaint_t sector, int byte_offset, int byte_len, char *buf); void ext4fs_set_blk_dev(struct blk_desc *rbdd, disk_partition_t *info); -long int read_allocated_block(struct ext2_inode *inode, int fileblock); +long int read_allocated_block(struct ext2_inode *inode, int fileblock, + struct ext_block_cache *cache); int ext4fs_probe(struct blk_desc *fs_dev_desc, disk_partition_t *fs_partition); int ext4_read_file(const char *filename, void *buf, loff_t offset, loff_t len, loff_t *actread); int ext4_read_superblock(char *buffer); int ext4fs_uuid(char *uuid_str); +void ext_cache_init(struct ext_block_cache *cache); +void ext_cache_fini(struct ext_block_cache *cache); +int ext_cache_read(struct ext_block_cache *cache, lbaint_t block, int size); #endif From 5cfc73e6e29f37a3a8daf5fd28bc4d5d015f778c Mon Sep 17 00:00:00 2001 From: Jean-Jacques Hiblot Date: Wed, 13 Feb 2019 12:15:22 +0100 Subject: [PATCH 02/16] test: fs: disable the metadata checksums on ext4 filesystems If the metadata checksums are enabled, all write operations will fail. Signed-off-by: Jean-Jacques Hiblot Reviewed-by: Tom Rini --- test/py/tests/test_fs/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/py/tests/test_fs/conftest.py b/test/py/tests/test_fs/conftest.py index 43eeb4be0b..745ed0ed38 100644 --- a/test/py/tests/test_fs/conftest.py +++ b/test/py/tests/test_fs/conftest.py @@ -143,6 +143,8 @@ def mk_fs(config, fs_type, size, id): mkfs_opt = '-F 16' elif fs_type == 'fat32': mkfs_opt = '-F 32' + elif fs_type == 'ext4': + mkfs_opt = '-O ^metadata_csum' else: mkfs_opt = '' From 290100583dc70185e76ddaa00c1a50abd162a5ad Mon Sep 17 00:00:00 2001 From: Jean-Jacques Hiblot Date: Wed, 13 Feb 2019 12:15:23 +0100 Subject: [PATCH 03/16] test: fs: Add filesystem integrity checks We need to make sure that file writes,file creation, etc. are properly performed and do not corrupt the filesystem. To help with this, introduce the assert_fs_integrity() function that executes the appropriate fsck tool. It should be called at the end of any test that modify the content/organization of the filesystem. Currently only supports FATs and EXT4. Signed-off-by: Jean-Jacques Hiblot Reviewed-by: Tom Rini --- test/py/tests/test_fs/fstest_helpers.py | 15 +++++++++++++++ test/py/tests/test_fs/test_basic.py | 4 ++++ test/py/tests/test_fs/test_ext.py | 10 ++++++++++ test/py/tests/test_fs/test_mkdir.py | 8 ++++++++ test/py/tests/test_fs/test_unlink.py | 14 +++++++++++--- 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 test/py/tests/test_fs/fstest_helpers.py diff --git a/test/py/tests/test_fs/fstest_helpers.py b/test/py/tests/test_fs/fstest_helpers.py new file mode 100644 index 0000000000..637bc30a91 --- /dev/null +++ b/test/py/tests/test_fs/fstest_helpers.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2019, Texas Instrument +# Author: JJ Hiblot +# + +from subprocess import check_call, CalledProcessError + +def assert_fs_integrity(fs_type, fs_img): + try: + if fs_type == 'ext4': + check_call('fsck.ext4 -n -f %s' % fs_img, shell=True) + if fs_type == 'fat': + check_call('fsck.fat -n %s' % fs_img, shell=True) + except CalledProcessError: + raise diff --git a/test/py/tests/test_fs/test_basic.py b/test/py/tests/test_fs/test_basic.py index 140ca29ac7..71f3e86fb1 100644 --- a/test/py/tests/test_fs/test_basic.py +++ b/test/py/tests/test_fs/test_basic.py @@ -11,6 +11,7 @@ This test verifies basic read/write operation on file system. import pytest import re from fstest_defs import * +from fstest_helpers import assert_fs_integrity @pytest.mark.boardspec('sandbox') @pytest.mark.slow @@ -237,6 +238,7 @@ class TestFsBasic(object): 'md5sum %x $filesize' % ADDR, 'setenv filesize']) assert(md5val[0] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs12(self, u_boot_console, fs_obj_basic): """ @@ -252,6 +254,7 @@ class TestFsBasic(object): 'host bind 0 %s' % fs_img, '%swrite host 0:0 %x /. 0x10' % (fs_type, ADDR)]) assert('Unable to write' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs13(self, u_boot_console, fs_obj_basic): """ @@ -286,3 +289,4 @@ class TestFsBasic(object): 'md5sum %x $filesize' % ADDR, 'setenv filesize']) assert(md5val[0] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) diff --git a/test/py/tests/test_fs/test_ext.py b/test/py/tests/test_fs/test_ext.py index 06cad5516d..2c47738b8d 100644 --- a/test/py/tests/test_fs/test_ext.py +++ b/test/py/tests/test_fs/test_ext.py @@ -11,6 +11,7 @@ This test verifies extended write operation on file system. import pytest import re from fstest_defs import * +from fstest_helpers import assert_fs_integrity @pytest.mark.boardspec('sandbox') @pytest.mark.slow @@ -36,6 +37,7 @@ class TestFsExt(object): 'md5sum %x $filesize' % ADDR, 'setenv filesize']) assert(md5val[0] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext2(self, u_boot_console, fs_obj_ext): """ @@ -58,6 +60,7 @@ class TestFsExt(object): 'md5sum %x $filesize' % ADDR, 'setenv filesize']) assert(md5val[0] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext3(self, u_boot_console, fs_obj_ext): """ @@ -72,6 +75,7 @@ class TestFsExt(object): '%swrite host 0:0 %x /dir1/none/%s.w3 $filesize' % (fs_type, ADDR, MIN_FILE)]) assert('Unable to write "/dir1/none/' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext4(self, u_boot_console, fs_obj_ext): """ @@ -104,6 +108,7 @@ class TestFsExt(object): 'md5sum %x $filesize' % ADDR, 'setenv filesize']) assert(md5val[1] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext5(self, u_boot_console, fs_obj_ext): """ @@ -136,6 +141,7 @@ class TestFsExt(object): 'md5sum %x $filesize' % ADDR, 'setenv filesize']) assert(md5val[2] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext6(self, u_boot_console, fs_obj_ext): """ @@ -160,6 +166,7 @@ class TestFsExt(object): 'printenv filesize', 'setenv filesize']) assert('filesize=0' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext7(self, u_boot_console, fs_obj_ext): """ @@ -192,6 +199,7 @@ class TestFsExt(object): 'md5sum %x $filesize' % ADDR, 'setenv filesize']) assert(md5val[3] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext8(self, u_boot_console, fs_obj_ext): """ @@ -209,6 +217,7 @@ class TestFsExt(object): '%swrite host 0:0 %x /dir1/%s.w8 0x1400 %x' % (fs_type, ADDR, MIN_FILE, 0x100000 + 0x1400)) assert('Unable to write "/dir1' in output) + assert_fs_integrity(fs_type, fs_img) def test_fs_ext9(self, u_boot_console, fs_obj_ext): """ @@ -223,3 +232,4 @@ class TestFsExt(object): '%swrite host 0:0 %x /dir1/%s.w9 0x1400 0x1400' % (fs_type, ADDR, MIN_FILE)]) assert('Unable to write "/dir1' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) diff --git a/test/py/tests/test_fs/test_mkdir.py b/test/py/tests/test_fs/test_mkdir.py index b3fe11cf3b..fa9561ec35 100644 --- a/test/py/tests/test_fs/test_mkdir.py +++ b/test/py/tests/test_fs/test_mkdir.py @@ -9,6 +9,7 @@ This test verifies mkdir operation on file system. """ import pytest +from fstest_helpers import assert_fs_integrity @pytest.mark.boardspec('sandbox') @pytest.mark.slow @@ -29,6 +30,8 @@ class TestMkdir(object): '%sls host 0:0 dir1' % fs_type) assert('./' in output) assert('../' in output) + assert_fs_integrity(fs_type, fs_img) + def test_mkdir2(self, u_boot_console, fs_obj_mkdir): """ @@ -46,6 +49,7 @@ class TestMkdir(object): '%sls host 0:0 dir1/dir2' % fs_type) assert('./' in output) assert('../' in output) + assert_fs_integrity(fs_type, fs_img) def test_mkdir3(self, u_boot_console, fs_obj_mkdir): """ @@ -58,6 +62,7 @@ class TestMkdir(object): 'host bind 0 %s' % fs_img, '%smkdir host 0:0 none/dir3' % fs_type]) assert('Unable to create a directory' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_mkdir4(self, u_boot_console, fs_obj_mkdir): """ @@ -69,6 +74,7 @@ class TestMkdir(object): 'host bind 0 %s' % fs_img, '%smkdir host 0:0 .' % fs_type]) assert('Unable to create a directory' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_mkdir5(self, u_boot_console, fs_obj_mkdir): """ @@ -80,6 +86,7 @@ class TestMkdir(object): 'host bind 0 %s' % fs_img, '%smkdir host 0:0 ..' % fs_type]) assert('Unable to create a directory' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_mkdir6(self, u_boot_console, fs_obj_mkdir): """ @@ -111,3 +118,4 @@ class TestMkdir(object): '%sls host 0:0 dir6/0123456789abcdef13/..' % fs_type) assert('0123456789abcdef00/' in output) assert('0123456789abcdef13/' in output) + assert_fs_integrity(fs_type, fs_img) diff --git a/test/py/tests/test_fs/test_unlink.py b/test/py/tests/test_fs/test_unlink.py index 2b817468ed..97aafc63bb 100644 --- a/test/py/tests/test_fs/test_unlink.py +++ b/test/py/tests/test_fs/test_unlink.py @@ -10,6 +10,7 @@ on file system. """ import pytest +from fstest_helpers import assert_fs_integrity @pytest.mark.boardspec('sandbox') @pytest.mark.slow @@ -30,6 +31,7 @@ class TestUnlink(object): '%sls host 0:0 dir1/' % fs_type) assert(not 'file1' in output) assert('file2' in output) + assert_fs_integrity(fs_type, fs_img) def test_unlink2(self, u_boot_console, fs_obj_unlink): """ @@ -48,6 +50,7 @@ class TestUnlink(object): output = u_boot_console.run_command( '%sls host 0:0 dir2' % fs_type) assert('0 file(s), 2 dir(s)' in output) + assert_fs_integrity(fs_type, fs_img) def test_unlink3(self, u_boot_console, fs_obj_unlink): """ @@ -59,6 +62,7 @@ class TestUnlink(object): 'host bind 0 %s' % fs_img, '%srm host 0:0 dir1/nofile' % fs_type]) assert('nofile: doesn\'t exist' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_unlink4(self, u_boot_console, fs_obj_unlink): """ @@ -71,9 +75,10 @@ class TestUnlink(object): '%srm host 0:0 dir4' % fs_type]) assert('' == ''.join(output)) - output = u_boot_console.run_command( - '%sls host 0:0 /' % fs_type) - assert(not 'dir4' in output) + output = u_boot_console.run_command( + '%sls host 0:0 /' % fs_type) + assert(not 'dir4' in output) + assert_fs_integrity(fs_type, fs_img) def test_unlink5(self, u_boot_console, fs_obj_unlink): """ @@ -86,6 +91,7 @@ class TestUnlink(object): 'host bind 0 %s' % fs_img, '%srm host 0:0 dir5' % fs_type]) assert('directory is not empty' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_unlink6(self, u_boot_console, fs_obj_unlink): """ @@ -97,6 +103,7 @@ class TestUnlink(object): 'host bind 0 %s' % fs_img, '%srm host 0:0 dir5/.' % fs_type]) assert('directory is not empty' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) def test_unlink7(self, u_boot_console, fs_obj_unlink): """ @@ -108,3 +115,4 @@ class TestUnlink(object): 'host bind 0 %s' % fs_img, '%srm host 0:0 dir5/..' % fs_type]) assert('directory is not empty' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) From b000180b0f467851525aae3d0dfb8ab3a9dbcf8f Mon Sep 17 00:00:00 2001 From: Jean-Jacques Hiblot Date: Wed, 13 Feb 2019 12:15:24 +0100 Subject: [PATCH 04/16] fs: ext4: constify the buffer passed to write functions There is no need to modify the buffer passed to ext4fs_write_file(). The memset() call is not required here and was likely copied from the equivalent part of the ext4fs_read_file() function where we do need it. Signed-off-by: Jean-Jacques Hiblot Reviewed-by: Tom Rini --- fs/ext4/ext4_common.c | 2 +- fs/ext4/ext4_common.h | 2 +- fs/ext4/ext4_write.c | 11 +++++------ include/ext4fs.h | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/ext4/ext4_common.c b/fs/ext4/ext4_common.c index 29308e3b44..e9123785f1 100644 --- a/fs/ext4/ext4_common.c +++ b/fs/ext4/ext4_common.c @@ -190,7 +190,7 @@ uint32_t ext4fs_div_roundup(uint32_t size, uint32_t n) return res; } -void put_ext4(uint64_t off, void *buf, uint32_t size) +void put_ext4(uint64_t off, const void *buf, uint32_t size) { uint64_t startblock; uint64_t remainder; diff --git a/fs/ext4/ext4_common.h b/fs/ext4/ext4_common.h index 1ee81ab7ce..4dff1914d9 100644 --- a/fs/ext4/ext4_common.h +++ b/fs/ext4/ext4_common.h @@ -72,7 +72,7 @@ int ext4fs_iget(int inode_no, struct ext2_inode *inode); void ext4fs_allocate_blocks(struct ext2_inode *file_inode, unsigned int total_remaining_blocks, unsigned int *total_no_of_block); -void put_ext4(uint64_t off, void *buf, uint32_t size); +void put_ext4(uint64_t off, const void *buf, uint32_t size); struct ext2_block_group *ext4fs_get_group_descriptor (const struct ext_filesystem *fs, uint32_t bg_idx); uint64_t ext4fs_bg_get_block_id(const struct ext2_block_group *bg, diff --git a/fs/ext4/ext4_write.c b/fs/ext4/ext4_write.c index 95ffa3dfad..b5b7ee8133 100644 --- a/fs/ext4/ext4_write.c +++ b/fs/ext4/ext4_write.c @@ -752,7 +752,7 @@ void ext4fs_deinit(void) * contigous sectors as ext4fs_read_file */ static int ext4fs_write_file(struct ext2_inode *file_inode, - int pos, unsigned int len, char *buf) + int pos, unsigned int len, const char *buf) { int i; int blockcnt; @@ -764,7 +764,7 @@ static int ext4fs_write_file(struct ext2_inode *file_inode, int delayed_start = 0; int delayed_extent = 0; int delayed_next = 0; - char *delayed_buf = NULL; + const char *delayed_buf = NULL; /* Adjust len so it we can't read past the end of the file. */ if (len > filesize) @@ -816,7 +816,6 @@ static int ext4fs_write_file(struct ext2_inode *file_inode, (uint32_t) delayed_extent); previous_block_number = -1; } - memset(buf, 0, fs->blksz - skipfirst); } buf += fs->blksz - skipfirst; } @@ -830,8 +829,8 @@ static int ext4fs_write_file(struct ext2_inode *file_inode, return len; } -int ext4fs_write(const char *fname, unsigned char *buffer, - unsigned long sizebytes) +int ext4fs_write(const char *fname, const char *buffer, + unsigned long sizebytes) { int ret = 0; struct ext2_inode *file_inode = NULL; @@ -949,7 +948,7 @@ int ext4fs_write(const char *fname, unsigned char *buffer, if (ext4fs_put_metadata(temp_ptr, itable_blkno)) goto fail; /* copy the file content into data blocks */ - if (ext4fs_write_file(file_inode, 0, sizebytes, (char *)buffer) == -1) { + if (ext4fs_write_file(file_inode, 0, sizebytes, buffer) == -1) { printf("Error in copying content\n"); /* FIXME: Deallocate data blocks */ goto fail; diff --git a/include/ext4fs.h b/include/ext4fs.h index 4b5de6e7b6..7d48b2bc46 100644 --- a/include/ext4fs.h +++ b/include/ext4fs.h @@ -134,7 +134,7 @@ extern int gindex; int ext4fs_init(void); void ext4fs_deinit(void); int ext4fs_filename_unlink(char *filename); -int ext4fs_write(const char *fname, unsigned char *buffer, +int ext4fs_write(const char *fname, const char *buffer, unsigned long sizebytes); int ext4_write_file(const char *filename, void *buf, loff_t offset, loff_t len, loff_t *actwrite); From 5efc0686eebc0c0daabfbfc2c403f8251b468526 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Hiblot Date: Wed, 13 Feb 2019 12:15:25 +0100 Subject: [PATCH 05/16] fs: ext4: Add support for the creation of symbolic links Re-use the functions used to write/create a file, to support creation of a symbolic link. The difference with a regular file are small: - The inode mode is flagged with S_IFLNK instead of S_IFREG - The ext2_dirent's filetype is FILETYPE_SYMLINK instead of FILETYPE_REG - Instead of storing the content of a file in allocated blocks, the path to the target is stored. And if the target's path is short enough, no block is allocated and the target's path is stored in ext2_inode.b.symlink As with regulars files, if a file/symlink with the same name exits, it is unlinked first and then re-created. Signed-off-by: Jean-Jacques Hiblot Reviewed-by: Tom Rini [trini: Fix ext4 env code] Signed-off-by: Tom Rini --- env/ext4.c | 2 +- fs/ext4/ext4_common.c | 2 +- fs/ext4/ext4_write.c | 51 ++++++++++++++++++++++++++++++++++++------- include/ext4fs.h | 3 ++- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/env/ext4.c b/env/ext4.c index 09c5e4a491..388474a11c 100644 --- a/env/ext4.c +++ b/env/ext4.c @@ -60,7 +60,7 @@ static int env_ext4_save(void) } err = ext4fs_write(CONFIG_ENV_EXT4_FILE, (void *)&env_new, - sizeof(env_t)); + sizeof(env_t), FILETYPE_REG); ext4fs_close(); if (err == -1) { diff --git a/fs/ext4/ext4_common.c b/fs/ext4/ext4_common.c index e9123785f1..59ad6c8f8c 100644 --- a/fs/ext4/ext4_common.c +++ b/fs/ext4/ext4_common.c @@ -608,7 +608,7 @@ restart_read: dir->direntlen = cpu_to_le16(fs->blksz - totalbytes); dir->namelen = strlen(filename); - dir->filetype = FILETYPE_REG; /* regular file */ + dir->filetype = file_type; temp_dir = (char *)dir; temp_dir = temp_dir + sizeof(struct ext2_dirent); memcpy(temp_dir, filename, strlen(filename)); diff --git a/fs/ext4/ext4_write.c b/fs/ext4/ext4_write.c index b5b7ee8133..504d23a895 100644 --- a/fs/ext4/ext4_write.c +++ b/fs/ext4/ext4_write.c @@ -465,6 +465,15 @@ static int ext4fs_delete_file(int inodeno) if (le32_to_cpu(inode.size) % fs->blksz) no_blocks++; + /* + * special case for symlinks whose target are small enough that + *it fits in struct ext2_inode.b.symlink: no block had been allocated + */ + if ((le16_to_cpu(inode.mode) & S_IFLNK) && + le32_to_cpu(inode.size) <= sizeof(inode.b.symlink)) { + no_blocks = 0; + } + if (le32_to_cpu(inode.flags) & EXT4_EXTENTS_FL) { /* FIXME delete extent index blocks, i.e. eh_depth >= 1 */ struct ext4_extent_header *eh = @@ -830,7 +839,7 @@ static int ext4fs_write_file(struct ext2_inode *file_inode, } int ext4fs_write(const char *fname, const char *buffer, - unsigned long sizebytes) + unsigned long sizebytes, int type) { int ret = 0; struct ext2_inode *file_inode = NULL; @@ -853,8 +862,12 @@ int ext4fs_write(const char *fname, const char *buffer, struct ext2_block_group *bgd = NULL; struct ext_filesystem *fs = get_fs(); ALLOC_CACHE_ALIGN_BUFFER(char, filename, 256); + bool store_link_in_inode = false; memset(filename, 0x00, 256); + if (type != FILETYPE_REG && type != FILETYPE_SYMLINK) + return -1; + g_parent_inode = zalloc(fs->inodesz); if (!g_parent_inode) goto fail; @@ -892,8 +905,16 @@ int ext4fs_write(const char *fname, const char *buffer, if (ret) goto fail; } - /* calucalate how many blocks required */ - bytes_reqd_for_file = sizebytes; + + /* calculate how many blocks required */ + if (type == FILETYPE_SYMLINK && + sizebytes <= sizeof(file_inode->b.symlink)) { + store_link_in_inode = true; + bytes_reqd_for_file = 0; + } else { + bytes_reqd_for_file = sizebytes; + } + blks_reqd_for_file = lldiv(bytes_reqd_for_file, fs->blksz); if (do_div(bytes_reqd_for_file, fs->blksz) != 0) { blks_reqd_for_file++; @@ -906,7 +927,7 @@ int ext4fs_write(const char *fname, const char *buffer, goto fail; } - inodeno = ext4fs_update_parent_dentry(filename, FILETYPE_REG); + inodeno = ext4fs_update_parent_dentry(filename, type); if (inodeno == -1) goto fail; /* prepare file inode */ @@ -914,14 +935,23 @@ int ext4fs_write(const char *fname, const char *buffer, if (!inode_buffer) goto fail; file_inode = (struct ext2_inode *)inode_buffer; - file_inode->mode = cpu_to_le16(S_IFREG | S_IRWXU | - S_IRGRP | S_IROTH | S_IXGRP | S_IXOTH); + file_inode->size = cpu_to_le32(sizebytes); + if (type == FILETYPE_SYMLINK) { + file_inode->mode = cpu_to_le16(S_IFLNK | S_IRWXU | S_IRWXG | + S_IRWXO); + if (store_link_in_inode) { + strncpy(file_inode->b.symlink, buffer, sizebytes); + sizebytes = 0; + } + } else { + file_inode->mode = cpu_to_le16(S_IFREG | S_IRWXU | S_IRGRP | + S_IROTH | S_IXGRP | S_IXOTH); + } /* ToDo: Update correct time */ file_inode->mtime = cpu_to_le32(timestamp); file_inode->atime = cpu_to_le32(timestamp); file_inode->ctime = cpu_to_le32(timestamp); file_inode->nlinks = cpu_to_le16(1); - file_inode->size = cpu_to_le32(sizebytes); /* Allocate data blocks */ ext4fs_allocate_blocks(file_inode, blocks_remaining, @@ -1013,7 +1043,7 @@ int ext4_write_file(const char *filename, void *buf, loff_t offset, return -1; } - ret = ext4fs_write(filename, buf, len); + ret = ext4fs_write(filename, buf, len, FILETYPE_REG); if (ret) { printf("** Error ext4fs_write() **\n"); goto fail; @@ -1028,3 +1058,8 @@ fail: return -1; } + +int ext4fs_create_link(const char *target, const char *fname) +{ + return ext4fs_write(fname, target, strlen(target), FILETYPE_SYMLINK); +} diff --git a/include/ext4fs.h b/include/ext4fs.h index 7d48b2bc46..34585d407d 100644 --- a/include/ext4fs.h +++ b/include/ext4fs.h @@ -135,9 +135,10 @@ int ext4fs_init(void); void ext4fs_deinit(void); int ext4fs_filename_unlink(char *filename); int ext4fs_write(const char *fname, const char *buffer, - unsigned long sizebytes); + unsigned long sizebytes, int type); int ext4_write_file(const char *filename, void *buf, loff_t offset, loff_t len, loff_t *actwrite); +int ext4fs_create_link(const char *target, const char *fname); #endif struct ext_filesystem *get_fs(void); From aaa12157c7d22132688ae97dcb35fc37f9ae88d5 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Hiblot Date: Wed, 13 Feb 2019 12:15:26 +0100 Subject: [PATCH 06/16] fs: Add a new command to create symbolic links The command line is: ln target linkname Currently symbolic links are supported only in ext4 and only if the option CMD_EXT4_WRITE is enabled. Signed-off-by: Jean-Jacques Hiblot Reviewed-by: Tom Rini --- cmd/fs.c | 14 ++++++++++++++ fs/fs.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ include/fs.h | 2 ++ 3 files changed, 60 insertions(+) diff --git a/cmd/fs.c b/cmd/fs.c index 94467671be..aaafbf9b52 100644 --- a/cmd/fs.c +++ b/cmd/fs.c @@ -76,6 +76,20 @@ U_BOOT_CMD( " device type 'interface' instance 'dev'." ) +static int do_ln_wrapper(cmd_tbl_t *cmdtp, int flag, int argc, + char * const argv[]) +{ + return do_ln(cmdtp, flag, argc, argv, FS_TYPE_ANY); +} + +U_BOOT_CMD( + ln, 5, 1, do_ln_wrapper, + "Create a symbolic link", + " target linkname\n" + " - create a symbolic link to 'target' with the name 'linkname' on\n" + " device type 'interface' instance 'dev'." +) + static int do_fstype_wrapper(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]) { diff --git a/fs/fs.c b/fs/fs.c index c5c35ebf5f..736ebef4a9 100644 --- a/fs/fs.c +++ b/fs/fs.c @@ -90,6 +90,11 @@ static inline int fs_write_unsupported(const char *filename, void *buf, return -1; } +static inline int fs_ln_unsupported(const char *filename, const char *target) +{ + return -1; +} + static inline void fs_close_unsupported(void) { } @@ -154,6 +159,7 @@ struct fstype_info { void (*closedir)(struct fs_dir_stream *dirs); int (*unlink)(const char *filename); int (*mkdir)(const char *dirname); + int (*ln)(const char *filename, const char *target); }; static struct fstype_info fstypes[] = { @@ -181,6 +187,7 @@ static struct fstype_info fstypes[] = { .opendir = fat_opendir, .readdir = fat_readdir, .closedir = fat_closedir, + .ln = fs_ln_unsupported, }, #endif @@ -197,8 +204,10 @@ static struct fstype_info fstypes[] = { .read = ext4_read_file, #ifdef CONFIG_CMD_EXT4_WRITE .write = ext4_write_file, + .ln = ext4fs_create_link, #else .write = fs_write_unsupported, + .ln = fs_ln_unsupported, #endif .uuid = ext4fs_uuid, .opendir = fs_opendir_unsupported, @@ -222,6 +231,7 @@ static struct fstype_info fstypes[] = { .opendir = fs_opendir_unsupported, .unlink = fs_unlink_unsupported, .mkdir = fs_mkdir_unsupported, + .ln = fs_ln_unsupported, }, #endif #ifdef CONFIG_CMD_UBIFS @@ -240,6 +250,7 @@ static struct fstype_info fstypes[] = { .opendir = fs_opendir_unsupported, .unlink = fs_unlink_unsupported, .mkdir = fs_mkdir_unsupported, + .ln = fs_ln_unsupported, }, #endif #ifdef CONFIG_FS_BTRFS @@ -258,6 +269,7 @@ static struct fstype_info fstypes[] = { .opendir = fs_opendir_unsupported, .unlink = fs_unlink_unsupported, .mkdir = fs_mkdir_unsupported, + .ln = fs_ln_unsupported, }, #endif { @@ -275,6 +287,7 @@ static struct fstype_info fstypes[] = { .opendir = fs_opendir_unsupported, .unlink = fs_unlink_unsupported, .mkdir = fs_mkdir_unsupported, + .ln = fs_ln_unsupported, }, }; @@ -602,6 +615,22 @@ int fs_mkdir(const char *dirname) return ret; } +int fs_ln(const char *fname, const char *target) +{ + struct fstype_info *info = fs_get_info(fs_type); + int ret; + + ret = info->ln(fname, target); + + if (ret < 0) { + printf("** Unable to create link %s -> %s **\n", fname, target); + ret = -1; + } + fs_close(); + + return ret; +} + int do_size(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[], int fstype) { @@ -840,3 +869,18 @@ int do_mkdir(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[], return 0; } + +int do_ln(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[], + int fstype) +{ + if (argc != 5) + return CMD_RET_USAGE; + + if (fs_set_blk_dev(argv[1], argv[2], fstype)) + return 1; + + if (fs_ln(argv[3], argv[4])) + return 1; + + return 0; +} diff --git a/include/fs.h b/include/fs.h index aa3604db8d..6854597700 100644 --- a/include/fs.h +++ b/include/fs.h @@ -191,6 +191,8 @@ int do_rm(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[], int fstype); int do_mkdir(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[], int fstype); +int do_ln(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[], + int fstype); /* * Determine the UUID of the specified filesystem and print it. Optionally it is From ef79284e7a6ba141b56478f8ed1456ef5c27ae47 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Hiblot Date: Wed, 13 Feb 2019 12:15:27 +0100 Subject: [PATCH 07/16] test: fs: Added tests for symlinks Test cases are: 1) basic link creation, verify it can be followed 2) chained links, verify it can be followed 3) replace exiting file a with a link, and a link with a link. verify it can be followed 4) create a broken link, verify it can't be followed Signed-off-by: Jean-Jacques Hiblot Reviewed-by: Tom Rini --- test/py/tests/test_fs/conftest.py | 75 +++++++++++++++ test/py/tests/test_fs/fstest_defs.py | 3 + test/py/tests/test_fs/test_symlink.py | 130 ++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 test/py/tests/test_fs/test_symlink.py diff --git a/test/py/tests/test_fs/conftest.py b/test/py/tests/test_fs/conftest.py index 745ed0ed38..9324657d21 100644 --- a/test/py/tests/test_fs/conftest.py +++ b/test/py/tests/test_fs/conftest.py @@ -13,6 +13,7 @@ supported_fs_basic = ['fat16', 'fat32', 'ext4'] supported_fs_ext = ['fat16', 'fat32'] supported_fs_mkdir = ['fat16', 'fat32'] supported_fs_unlink = ['fat16', 'fat32'] +supported_fs_symlink = ['ext4'] # # Filesystem test specific setup @@ -48,6 +49,7 @@ def pytest_configure(config): global supported_fs_ext global supported_fs_mkdir global supported_fs_unlink + global supported_fs_symlink def intersect(listA, listB): return [x for x in listA if x in listB] @@ -59,6 +61,7 @@ def pytest_configure(config): supported_fs_ext = intersect(supported_fs, supported_fs_ext) supported_fs_mkdir = intersect(supported_fs, supported_fs_mkdir) supported_fs_unlink = intersect(supported_fs, supported_fs_unlink) + supported_fs_symlink = intersect(supported_fs, supported_fs_symlink) def pytest_generate_tests(metafunc): """Parametrize fixtures, fs_obj_xxx @@ -84,6 +87,9 @@ def pytest_generate_tests(metafunc): if 'fs_obj_unlink' in metafunc.fixturenames: metafunc.parametrize('fs_obj_unlink', supported_fs_unlink, indirect=True, scope='module') + if 'fs_obj_symlink' in metafunc.fixturenames: + metafunc.parametrize('fs_obj_symlink', supported_fs_symlink, + indirect=True, scope='module') # # Helper functions @@ -525,3 +531,72 @@ def fs_obj_unlink(request, u_boot_config): call('rmdir %s' % mount_dir, shell=True) if fs_img: call('rm -f %s' % fs_img, shell=True) + +# +# Fixture for symlink fs test +# +# NOTE: yield_fixture was deprecated since pytest-3.0 +@pytest.yield_fixture() +def fs_obj_symlink(request, u_boot_config): + """Set up a file system to be used in symlink fs test. + + Args: + request: Pytest request object. + u_boot_config: U-boot configuration. + + Return: + A fixture for basic fs test, i.e. a triplet of file system type, + volume file name and a list of MD5 hashes. + """ + fs_type = request.param + fs_img = '' + + fs_ubtype = fstype_to_ubname(fs_type) + check_ubconfig(u_boot_config, fs_ubtype) + + mount_dir = u_boot_config.persistent_data_dir + '/mnt' + + small_file = mount_dir + '/' + SMALL_FILE + medium_file = mount_dir + '/' + MEDIUM_FILE + + try: + + # 3GiB volume + fs_img = mk_fs(u_boot_config, fs_type, 0x40000000, '1GB') + + # Mount the image so we can populate it. + check_call('mkdir -p %s' % mount_dir, shell=True) + mount_fs(fs_type, fs_img, mount_dir) + + # Create a subdirectory. + check_call('mkdir %s/SUBDIR' % mount_dir, shell=True) + + # Create a small file in this image. + check_call('dd if=/dev/urandom of=%s bs=1M count=1' + % small_file, shell=True) + + # Create a medium file in this image. + check_call('dd if=/dev/urandom of=%s bs=10M count=1' + % medium_file, shell=True) + + # Generate the md5sums of reads that we will test against small file + out = check_output( + 'dd if=%s bs=1M skip=0 count=1 2> /dev/null | md5sum' + % small_file, shell=True) + md5val = [out.split()[0]] + out = check_output( + 'dd if=%s bs=10M skip=0 count=1 2> /dev/null | md5sum' + % medium_file, shell=True) + md5val.extend([out.split()[0]]) + + umount_fs(mount_dir) + except CalledProcessError: + pytest.skip('Setup failed for filesystem: ' + fs_type) + return + else: + yield [fs_ubtype, fs_img, md5val] + finally: + umount_fs(mount_dir) + call('rmdir %s' % mount_dir, shell=True) + if fs_img: + call('rm -f %s' % fs_img, shell=True) diff --git a/test/py/tests/test_fs/fstest_defs.py b/test/py/tests/test_fs/fstest_defs.py index 5f107562d9..35b2bb6518 100644 --- a/test/py/tests/test_fs/fstest_defs.py +++ b/test/py/tests/test_fs/fstest_defs.py @@ -6,6 +6,9 @@ MIN_FILE='testfile' # $SMALL_FILE is the name of the 1MB file in the file system image SMALL_FILE='1MB.file' +# $MEDIUM_FILE is the name of the 10MB file in the file system image +MEDIUM_FILE='10MB.file' + # $BIG_FILE is the name of the 2.5GB file in the file system image BIG_FILE='2.5GB.file' diff --git a/test/py/tests/test_fs/test_symlink.py b/test/py/tests/test_fs/test_symlink.py new file mode 100644 index 0000000000..9ced101a29 --- /dev/null +++ b/test/py/tests/test_fs/test_symlink.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2019, Texas Instrument +# Author: Jean-Jacques Hiblot +# +# U-Boot File System:symlink Test + +""" +This test verifies unlink operation (deleting a file or a directory) +on file system. +""" + +import pytest +import re +from fstest_defs import * +from fstest_helpers import assert_fs_integrity + + +@pytest.mark.boardspec('sandbox') +@pytest.mark.slow +class TestSymlink(object): + def test_symlink1(self, u_boot_console, fs_obj_symlink): + """ + Test Case 1 - create a link. and follow it when reading + """ + fs_type, fs_img, md5val = fs_obj_symlink + with u_boot_console.log.section('Test Case 1 - create link and read'): + output = u_boot_console.run_command_list([ + 'host bind 0 %s' % fs_img, + 'setenv filesize', + 'ln host 0:0 %s /%s.link ' % (SMALL_FILE, SMALL_FILE), + ]) + assert('' in ''.join(output)) + + output = u_boot_console.run_command_list([ + '%sload host 0:0 %x /%s.link' % (fs_type, ADDR, SMALL_FILE), + 'printenv filesize']) + assert('filesize=100000' in ''.join(output)) + + # Test Case 4b - Read full 1MB of small file + output = u_boot_console.run_command_list([ + 'md5sum %x $filesize' % ADDR, + 'setenv filesize']) + assert(md5val[0] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) + + def test_symlink2(self, u_boot_console, fs_obj_symlink): + """ + Test Case 2 - create chained links + """ + fs_type, fs_img, md5val = fs_obj_symlink + with u_boot_console.log.section('Test Case 2 - create chained links'): + output = u_boot_console.run_command_list([ + 'host bind 0 %s' % fs_img, + 'setenv filesize', + 'ln host 0:0 %s /%s.link1 ' % (SMALL_FILE, SMALL_FILE), + 'ln host 0:0 /%s.link1 /SUBDIR/%s.link2' % ( + SMALL_FILE, SMALL_FILE), + 'ln host 0:0 SUBDIR/%s.link2 /%s.link3' % ( + SMALL_FILE, SMALL_FILE), + ]) + assert('' in ''.join(output)) + + output = u_boot_console.run_command_list([ + '%sload host 0:0 %x /%s.link3' % (fs_type, ADDR, SMALL_FILE), + 'printenv filesize']) + assert('filesize=100000' in ''.join(output)) + + # Test Case 4b - Read full 1MB of small file + output = u_boot_console.run_command_list([ + 'md5sum %x $filesize' % ADDR, + 'setenv filesize']) + assert(md5val[0] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) + + def test_symlink3(self, u_boot_console, fs_obj_symlink): + """ + Test Case 3 - replace file/link with link + """ + fs_type, fs_img, md5val = fs_obj_symlink + with u_boot_console.log.section('Test Case 1 - create link and read'): + output = u_boot_console.run_command_list([ + 'host bind 0 %s' % fs_img, + 'setenv filesize', + 'ln host 0:0 %s /%s ' % (MEDIUM_FILE, SMALL_FILE), + 'ln host 0:0 %s /%s.link ' % (MEDIUM_FILE, MEDIUM_FILE), + ]) + assert('' in ''.join(output)) + + output = u_boot_console.run_command_list([ + '%sload host 0:0 %x /%s' % (fs_type, ADDR, SMALL_FILE), + 'printenv filesize']) + assert('filesize=a00000' in ''.join(output)) + + output = u_boot_console.run_command_list([ + 'md5sum %x $filesize' % ADDR, + 'setenv filesize']) + assert(md5val[1] in ''.join(output)) + + output = u_boot_console.run_command_list([ + 'ln host 0:0 %s.link /%s ' % (MEDIUM_FILE, SMALL_FILE), + '%sload host 0:0 %x /%s' % (fs_type, ADDR, SMALL_FILE), + 'printenv filesize']) + assert('filesize=a00000' in ''.join(output)) + + output = u_boot_console.run_command_list([ + 'md5sum %x $filesize' % ADDR, + 'setenv filesize']) + assert(md5val[1] in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) + + def test_symlink4(self, u_boot_console, fs_obj_symlink): + """ + Test Case 4 - create a broken link + """ + fs_type, fs_img, md5val = fs_obj_symlink + with u_boot_console.log.section('Test Case 1 - create link and read'): + + output = u_boot_console.run_command_list([ + 'setenv filesize', + 'ln host 0:0 nowhere /link ', + ]) + assert('' in ''.join(output)) + + output = u_boot_console.run_command( + '%sload host 0:0 %x /link' % + (fs_type, ADDR)) + with u_boot_console.disable_check('error_notification'): + output = u_boot_console.run_command('printenv filesize') + assert('"filesize" not defined' in ''.join(output)) + assert_fs_integrity(fs_type, fs_img) From 1c48fda3e5a88159130b4d4805fbdf367212afab Mon Sep 17 00:00:00 2001 From: Gero Schumacher Date: Tue, 26 Feb 2019 15:45:22 +0000 Subject: [PATCH 08/16] fs: ext4: Problem with ext4load and sparse files Hi, when I try to load a sparse file via ext4load, I am getting the error message 'invalid extent' After a deeper look in the code, it seems to be an issue in the function ext4fs_get_extent_block in fs/ext4/ext4_common.c: The file starts with 1k of zeros. The blocksize is 1024. So the first extend block contains the following information: eh_entries: 1 eh_depth: 1 ei_block 1 When the upper layer (ext4fs_read_file) asks for fileblock 0, we are running in the 'invalid extent' error message. For me it seems, that the code is not prepared for handling a sparse block at the beginning of the file. The following change, solved my problem: I am really not an expert in ext4 filesystems. Can somebody please have a look at this issue and give me a feedback, if I am totally wrong or not? --- fs/ext4/ext4_common.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/ext4/ext4_common.c b/fs/ext4/ext4_common.c index 59ad6c8f8c..feffbfa9a9 100644 --- a/fs/ext4/ext4_common.c +++ b/fs/ext4/ext4_common.c @@ -1547,8 +1547,12 @@ static struct ext4_extent_header *ext4fs_get_extent_block break; } while (fileblock >= le32_to_cpu(index[i].ei_block)); - if (--i < 0) - return NULL; + /* + * If first logical block number is higher than requested fileblock, + * it is a sparse file. This is handled on upper layer. + */ + if (i > 0) + i--; block = le16_to_cpu(index[i].ei_leaf_hi); block = (block << 32) + le32_to_cpu(index[i].ei_leaf_lo); From 9b18358dc05df1dc69c2274a61ec6bcf75a058ad Mon Sep 17 00:00:00 2001 From: Anssi Hannula Date: Wed, 27 Feb 2019 12:55:57 +0200 Subject: [PATCH 09/16] fs: fat: fix reading non-cluster-aligned root directory A FAT12/FAT16 root directory location is specified by a sector offset and it might not start at a cluster boundary. It also resides before the data area (before cluster 2). However, the current code assumes that the root directory is located at a beginning of a cluster, causing no files to be found if that is not the case. Since the FAT12/FAT16 root directory is located before the data area and is not aligned to clusters, using unsigned cluster numbers to refer to the root directory does not work well (the "cluster number" may be negative, and even allowing it be signed would not make it properly aligned). Modify the code to not use the normal cluster numbering when referring to the root directory of FAT12/FAT16 and instead use a cluster-sized offsets counted from the root directory start sector. This is a relatively common case as at least the filesystem formatter on Win7 seems to create such filesystems by default on 2GB USB sticks when "FAT" is selected (cluster size 64 sectors, rootdir size 32 sectors, rootdir starts at half a cluster before cluster 2). dosfstools mkfs.vfat does not seem to create affected filesystems. Signed-off-by: Anssi Hannula Reviewed-by: Bernhard Messerklinger Tested-by: Bernhard Messerklinger --- fs/fat/fat.c | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/fs/fat/fat.c b/fs/fat/fat.c index 6ade4ea54e..c5997c2173 100644 --- a/fs/fat/fat.c +++ b/fs/fat/fat.c @@ -602,8 +602,13 @@ static int get_fs_info(fsdata *mydata) mydata->data_begin = mydata->rootdir_sect + mydata->rootdir_size - (mydata->clust_size * 2); - mydata->root_cluster = - sect_to_clust(mydata, mydata->rootdir_sect); + + /* + * The root directory is not cluster-aligned and may be on a + * "negative" cluster, this will be handled specially in + * next_cluster(). + */ + mydata->root_cluster = 0; } mydata->fatbufnum = -1; @@ -733,20 +738,38 @@ static void fat_itr_child(fat_itr *itr, fat_itr *parent) itr->last_cluster = 0; } -static void *next_cluster(fat_itr *itr) +static void *next_cluster(fat_itr *itr, unsigned *nbytes) { fsdata *mydata = itr->fsdata; /* for silly macros */ int ret; u32 sect; + u32 read_size; /* have we reached the end? */ if (itr->last_cluster) return NULL; - sect = clust_to_sect(itr->fsdata, itr->next_clust); + if (itr->is_root && itr->fsdata->fatsize != 32) { + /* + * The root directory is located before the data area and + * cannot be indexed using the regular unsigned cluster + * numbers (it may start at a "negative" cluster or not at a + * cluster boundary at all), so consider itr->next_clust to be + * a offset in cluster-sized units from the start of rootdir. + */ + unsigned sect_offset = itr->next_clust * itr->fsdata->clust_size; + unsigned remaining_sects = itr->fsdata->rootdir_size - sect_offset; + sect = itr->fsdata->rootdir_sect + sect_offset; + /* do not read past the end of rootdir */ + read_size = min_t(u32, itr->fsdata->clust_size, + remaining_sects); + } else { + sect = clust_to_sect(itr->fsdata, itr->next_clust); + read_size = itr->fsdata->clust_size; + } - debug("FAT read(sect=%d), clust_size=%d, DIRENTSPERBLOCK=%zd\n", - sect, itr->fsdata->clust_size, DIRENTSPERBLOCK); + debug("FAT read(sect=%d), clust_size=%d, read_size=%u, DIRENTSPERBLOCK=%zd\n", + sect, itr->fsdata->clust_size, read_size, DIRENTSPERBLOCK); /* * NOTE: do_fat_read_at() had complicated logic to deal w/ @@ -757,18 +780,17 @@ static void *next_cluster(fat_itr *itr) * dent at a time and iteratively constructing the vfat long * name. */ - ret = disk_read(sect, itr->fsdata->clust_size, - itr->block); + ret = disk_read(sect, read_size, itr->block); if (ret < 0) { debug("Error: reading block\n"); return NULL; } + *nbytes = read_size * itr->fsdata->sect_size; itr->clust = itr->next_clust; if (itr->is_root && itr->fsdata->fatsize != 32) { itr->next_clust++; - sect = clust_to_sect(itr->fsdata, itr->next_clust); - if (sect - itr->fsdata->rootdir_sect >= + if (itr->next_clust * itr->fsdata->clust_size >= itr->fsdata->rootdir_size) { debug("nextclust: 0x%x\n", itr->next_clust); itr->last_cluster = 1; @@ -787,9 +809,8 @@ static void *next_cluster(fat_itr *itr) static dir_entry *next_dent(fat_itr *itr) { if (itr->remaining == 0) { - struct dir_entry *dent = next_cluster(itr); - unsigned nbytes = itr->fsdata->sect_size * - itr->fsdata->clust_size; + unsigned nbytes; + struct dir_entry *dent = next_cluster(itr, &nbytes); /* have we reached the last cluster? */ if (!dent) { From d22f7ad86a3edfbfadbdfc11c754a4de5445ab39 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 13 Mar 2019 17:49:25 +0100 Subject: [PATCH 10/16] test/py: mmc: Factor out device selection Factor out the 'mmc dev' call so it can be recycled by other tests. Signed-off-by: Marek Vasut Cc: Tom Rini Cc: Simon Glass Reviewed-by: Simon Glass --- test/py/tests/test_mmc_rd.py | 38 +++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/test/py/tests/test_mmc_rd.py b/test/py/tests/test_mmc_rd.py index a13bc0a5f6..40ae1a8459 100644 --- a/test/py/tests/test_mmc_rd.py +++ b/test/py/tests/test_mmc_rd.py @@ -57,6 +57,32 @@ env__mmc_rd_configs = ( ) """ +def mmc_dev(u_boot_console, is_emmc, devid, partid): + """Run the "mmc dev" command. + + Args: + u_boot_console: A U-Boot console connection. + is_emmc: Whether the device is eMMC + devid: Device ID + partid: Partition ID + + Returns: + Nothing. + """ + + # Select MMC device + cmd = 'mmc dev %d' % devid + if is_emmc: + cmd += ' %d' % partid + response = u_boot_console.run_command(cmd) + assert 'no card present' not in response + if is_emmc: + partid_response = '(part %d)' % partid + else: + partid_response = '' + good_response = 'mmc%d%s is current device' % (devid, partid_response) + assert good_response in response + @pytest.mark.buildconfigspec('cmd_mmc') def test_mmc_rd(u_boot_console, env__mmc_rd_config): """Test the "mmc read" command. @@ -86,17 +112,7 @@ def test_mmc_rd(u_boot_console, env__mmc_rd_config): addr = '0x%08x' % ram_base # Select MMC device - cmd = 'mmc dev %d' % devid - if is_emmc: - cmd += ' %d' % partid - response = u_boot_console.run_command(cmd) - assert 'no card present' not in response - if is_emmc: - partid_response = '(part %d)' % partid - else: - partid_response = '' - good_response = 'mmc%d%s is current device' % (devid, partid_response) - assert good_response in response + mmc_dev(u_boot_console, is_emmc, devid, partid) # Clear target RAM if expected_crc32: From 86dfd152c917e063bec78b6e2327550b2b738924 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 13 Mar 2019 17:49:26 +0100 Subject: [PATCH 11/16] test/py: mmc: Add 'mmc dev' test Add separate test for 'mmc dev' subcommand. This tests whether the system can switch to a specific card. Signed-off-by: Marek Vasut Cc: Tom Rini Cc: Simon Glass Reviewed-by: Simon Glass --- test/py/tests/test_mmc_rd.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/py/tests/test_mmc_rd.py b/test/py/tests/test_mmc_rd.py index 40ae1a8459..1337074d3f 100644 --- a/test/py/tests/test_mmc_rd.py +++ b/test/py/tests/test_mmc_rd.py @@ -83,6 +83,27 @@ def mmc_dev(u_boot_console, is_emmc, devid, partid): good_response = 'mmc%d%s is current device' % (devid, partid_response) assert good_response in response +@pytest.mark.buildconfigspec('cmd_mmc') +def test_mmc_dev(u_boot_console, env__mmc_rd_config): + """Test the "mmc dev" command. + + Args: + u_boot_console: A U-Boot console connection. + env__mmc_rd_config: The single MMC configuration on which + to run the test. See the file-level comment above for details + of the format. + + Returns: + Nothing. + """ + + is_emmc = env__mmc_rd_config['is_emmc'] + devid = env__mmc_rd_config['devid'] + partid = env__mmc_rd_config.get('partid', 0) + + # Select MMC device + mmc_dev(u_boot_console, is_emmc, devid, partid) + @pytest.mark.buildconfigspec('cmd_mmc') def test_mmc_rd(u_boot_console, env__mmc_rd_config): """Test the "mmc read" command. From ce4b2cafa79c668c81c98d17725d4f4ffbbb2c21 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 13 Mar 2019 17:49:27 +0100 Subject: [PATCH 12/16] test/py: mmc: Add 'mmc rescan' test Add test for 'mmc rescan' subcommand. This tests whether the system can switch to a specific card and then rescan the card. Signed-off-by: Marek Vasut Cc: Tom Rini Cc: Simon Glass Reviewed-by: Simon Glass --- test/py/tests/test_mmc_rd.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/py/tests/test_mmc_rd.py b/test/py/tests/test_mmc_rd.py index 1337074d3f..86a57aafe5 100644 --- a/test/py/tests/test_mmc_rd.py +++ b/test/py/tests/test_mmc_rd.py @@ -104,6 +104,32 @@ def test_mmc_dev(u_boot_console, env__mmc_rd_config): # Select MMC device mmc_dev(u_boot_console, is_emmc, devid, partid) +@pytest.mark.buildconfigspec('cmd_mmc') +def test_mmc_rescan(u_boot_console, env__mmc_rd_config): + """Test the "mmc rescan" command. + + Args: + u_boot_console: A U-Boot console connection. + env__mmc_rd_config: The single MMC configuration on which + to run the test. See the file-level comment above for details + of the format. + + Returns: + Nothing. + """ + + is_emmc = env__mmc_rd_config['is_emmc'] + devid = env__mmc_rd_config['devid'] + partid = env__mmc_rd_config.get('partid', 0) + + # Select MMC device + mmc_dev(u_boot_console, is_emmc, devid, partid) + + # Rescan MMC device + cmd = 'mmc rescan' + response = u_boot_console.run_command(cmd) + assert 'no card present' not in response + @pytest.mark.buildconfigspec('cmd_mmc') def test_mmc_rd(u_boot_console, env__mmc_rd_config): """Test the "mmc read" command. From 4ffec8cdf5128b4333c16088f6d8cedb2b1fe285 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 13 Mar 2019 17:49:28 +0100 Subject: [PATCH 13/16] test/py: mmc: Add 'mmc info' test Add test for 'mmc info' subcommand. This tests whether the card information is obtained correctly and verifies the device, bus speed, bus mode and bus width. Signed-off-by: Marek Vasut Cc: Tom Rini Cc: Simon Glass Reviewed-by: Simon Glass --- test/py/tests/test_mmc_rd.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/py/tests/test_mmc_rd.py b/test/py/tests/test_mmc_rd.py index 86a57aafe5..aba512adfe 100644 --- a/test/py/tests/test_mmc_rd.py +++ b/test/py/tests/test_mmc_rd.py @@ -130,6 +130,43 @@ def test_mmc_rescan(u_boot_console, env__mmc_rd_config): response = u_boot_console.run_command(cmd) assert 'no card present' not in response +@pytest.mark.buildconfigspec('cmd_mmc') +def test_mmc_info(u_boot_console, env__mmc_rd_config): + """Test the "mmc info" command. + + Args: + u_boot_console: A U-Boot console connection. + env__mmc_rd_config: The single MMC configuration on which + to run the test. See the file-level comment above for details + of the format. + + Returns: + Nothing. + """ + + is_emmc = env__mmc_rd_config['is_emmc'] + devid = env__mmc_rd_config['devid'] + partid = env__mmc_rd_config.get('partid', 0) + info_device = env__mmc_rd_config['info_device'] + info_speed = env__mmc_rd_config['info_speed'] + info_mode = env__mmc_rd_config['info_mode'] + info_buswidth = env__mmc_rd_config['info_buswidth'] + + # Select MMC device + mmc_dev(u_boot_console, is_emmc, devid, partid) + + # Read MMC device information + cmd = 'mmc info' + response = u_boot_console.run_command(cmd) + good_response = "Device: %s" % info_device + assert good_response in response + good_response = "Bus Speed: %s" % info_speed + assert good_response in response + good_response = "Mode : %s" % info_mode + assert good_response in response + good_response = "Bus Width: %s" % info_buswidth + assert good_response in response + @pytest.mark.buildconfigspec('cmd_mmc') def test_mmc_rd(u_boot_console, env__mmc_rd_config): """Test the "mmc read" command. From e551979790e38469dfc29100e2d2dd64f3bcff38 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 13 Mar 2019 17:49:29 +0100 Subject: [PATCH 14/16] test/py: mmc: Add 'mmc read' performance check Add option to the mmc rd test to check the duration of the execution of the mmc read command. This allows intercepting read performance regressions. Signed-off-by: Marek Vasut Cc: Tom Rini Cc: Simon Glass Reviewed-by: Simon Glass --- test/py/tests/test_mmc_rd.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/py/tests/test_mmc_rd.py b/test/py/tests/test_mmc_rd.py index aba512adfe..2dc715bb51 100644 --- a/test/py/tests/test_mmc_rd.py +++ b/test/py/tests/test_mmc_rd.py @@ -6,6 +6,7 @@ # read if the test configuration contains a CRC of the expected data. import pytest +import time import u_boot_utils """ @@ -187,6 +188,7 @@ def test_mmc_rd(u_boot_console, env__mmc_rd_config): sector = env__mmc_rd_config.get('sector', 0) count_sectors = env__mmc_rd_config.get('count', 1) expected_crc32 = env__mmc_rd_config.get('crc32', None) + read_duration_max = env__mmc_rd_config.get('read_duration_max', 0) count_bytes = count_sectors * 512 bcfg = u_boot_console.config.buildconfig @@ -213,7 +215,9 @@ def test_mmc_rd(u_boot_console, env__mmc_rd_config): # Read data cmd = 'mmc read %s %x %x' % (addr, sector, count_sectors) + tstart = time.time() response = u_boot_console.run_command(cmd) + tend = time.time() good_response = 'MMC read: dev # %d, block # %d, count %d ... %d blocks read: OK' % ( devid, sector, count_sectors, count_sectors) assert good_response in response @@ -226,3 +230,10 @@ def test_mmc_rd(u_boot_console, env__mmc_rd_config): assert expected_crc32 in response else: u_boot_console.log.warning('CONFIG_CMD_CRC32 != y: Skipping check') + + # Check if the command did not take too long + if read_duration_max: + elapsed = tend - tstart + u_boot_console.log.info('Reading %d bytes took %f seconds' % + (count_bytes, elapsed)) + assert elapsed <= (read_duration_max - 0.01) From febbc583319b567fe3d83e521cc2ace9be8d1501 Mon Sep 17 00:00:00 2001 From: Benjamin Lim Date: Fri, 29 Mar 2019 07:29:45 -0400 Subject: [PATCH 15/16] Fix ext4 block group descriptor sizing Ext4 allows for arbitrarily sized block group descriptors when 64-bit addressing is enabled, which was previously not properly supported. This patch dynamically allocates a chunk of memory of the correct size. Signed-off-by: Benjamin Lim --- fs/ext4/ext4_common.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/fs/ext4/ext4_common.c b/fs/ext4/ext4_common.c index feffbfa9a9..464c33d0d7 100644 --- a/fs/ext4/ext4_common.c +++ b/fs/ext4/ext4_common.c @@ -1587,7 +1587,7 @@ static int ext4fs_blockgroup int ext4fs_read_inode(struct ext2_data *data, int ino, struct ext2_inode *inode) { - struct ext2_block_group blkgrp; + struct ext2_block_group *blkgrp; struct ext2_sblock *sblock = &data->sblock; struct ext_filesystem *fs = get_fs(); int log2blksz = get_fs()->dev_desc->log2blksz; @@ -1595,17 +1595,28 @@ int ext4fs_read_inode(struct ext2_data *data, int ino, struct ext2_inode *inode) long int blkno; unsigned int blkoff; + /* Allocate blkgrp based on gdsize (for 64-bit support). */ + blkgrp = zalloc(get_fs()->gdsize); + if (!blkgrp) + return 0; + /* It is easier to calculate if the first inode is 0. */ ino--; status = ext4fs_blockgroup(data, ino / le32_to_cpu - (sblock->inodes_per_group), &blkgrp); - if (status == 0) + (sblock->inodes_per_group), blkgrp); + if (status == 0) { + free(blkgrp); return 0; + } inodes_per_block = EXT2_BLOCK_SIZE(data) / fs->inodesz; - blkno = ext4fs_bg_get_inode_table_id(&blkgrp, fs) + + blkno = ext4fs_bg_get_inode_table_id(blkgrp, fs) + (ino % le32_to_cpu(sblock->inodes_per_group)) / inodes_per_block; blkoff = (ino % inodes_per_block) * fs->inodesz; + + /* Free blkgrp as it is no longer required. */ + free(blkgrp); + /* Read the inode. */ status = ext4fs_devread((lbaint_t)blkno << (LOG2_BLOCK_SIZE(data) - log2blksz), blkoff, From 0a8406602ae5f234f3f670ab59628673cc52ff4d Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 9 Apr 2019 16:08:52 -0400 Subject: [PATCH 16/16] test.py: Disable fsck for FAT tests for now Currently enabling fsck on FAT16/FAT32 exposes that we have problems with: TestFsBasic.test_fs13[fat16] TestFsBasic.test_fs11[fat32] TestFsBasic.test_fs12[fat32] TestFsBasic.test_fs13[fat32] TestFsExt.test_fs_ext1[fat32] TestFsExt.test_fs_ext2[fat32] TestFsExt.test_fs_ext3[fat32] TestFsExt.test_fs_ext4[fat32] TestFsExt.test_fs_ext5[fat32] TestFsExt.test_fs_ext6[fat32] TestFsExt.test_fs_ext7[fat32] TestFsExt.test_fs_ext8[fat32] TestFsExt.test_fs_ext9[fat32] TestMkdir.test_mkdir6[fat16] TestMkdir.test_mkdir1[fat32] TestMkdir.test_mkdir2[fat32] TestMkdir.test_mkdir3[fat32] TestMkdir.test_mkdir4[fat32] TestMkdir.test_mkdir5[fat32] TestMkdir.test_mkdir6[fat32] TestUnlink.test_unlink1[fat16] TestUnlink.test_unlink2[fat16] TestUnlink.test_unlink3[fat16] TestUnlink.test_unlink4[fat16] TestUnlink.test_unlink5[fat16] TestUnlink.test_unlink6[fat16] TestUnlink.test_unlink7[fat16] TestUnlink.test_unlink1[fat32] TestUnlink.test_unlink2[fat32] TestUnlink.test_unlink3[fat32] TestUnlink.test_unlink4[fat32] TestUnlink.test_unlink5[fat32] TestUnlink.test_unlink6[fat32] TestUnlink.test_unlink7[fat32] This is because we don't update the "information sector" on FAT32. While in the future we should resolve this problem and include that feature, we should enable fsck for ext4 to ensure that things remain in good shape there. Signed-off-by: Tom Rini --- test/py/tests/test_fs/fstest_helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/py/tests/test_fs/fstest_helpers.py b/test/py/tests/test_fs/fstest_helpers.py index 637bc30a91..faec298248 100644 --- a/test/py/tests/test_fs/fstest_helpers.py +++ b/test/py/tests/test_fs/fstest_helpers.py @@ -9,7 +9,5 @@ def assert_fs_integrity(fs_type, fs_img): try: if fs_type == 'ext4': check_call('fsck.ext4 -n -f %s' % fs_img, shell=True) - if fs_type == 'fat': - check_call('fsck.fat -n %s' % fs_img, shell=True) except CalledProcessError: raise