djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

extmod/vfs_lfs: Add mtime support to littlefs files. 

This commit adds support for modification time of files on littlefs v2
filesystems, using file attributes.  For some background see issue #6114.

Features/properties of this implementation:
- Only supported on littlefs2 (not littlefs1).
- Uses littlefs2's general file attributes to store the timestamp.
- The timestamp is 64-bits and stores nanoseconds since 1970/1/1 (if the
  range to the year 2554 is not enough then additional bits can be added to
  this timestamp by adding another file attribute).
- mtime is enabled by default but can be disabled in the constructor, eg:
  uos.mount(uos.VfsLfs2(bdev, mtime=False), '/flash')
- It's fully backwards compatible, existing littlefs2 filesystems will work
  without reformatting and timestamps will be added transparently to
  existing files (once they are opened for writing).
- Files without timestamps will open correctly, and stat will just return 0
  for their timestamp.
- mtime can be disabled or enabled each mount time and timestamps will only
  be updated if mtime is enabled (otherwise they will be untouched).

Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Damien George 2020-07-29 01:01:48 +10:00
parent ee50a6effe
commit 2acc087880
8 changed files with 197 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/library/uos.rst
View File

@ -178,7 +178,7 @@ represented by VFS classes.
Build a FAT filesystem on *block_dev*.
.. class:: VfsLfs1(block_dev)
.. class:: VfsLfs1(block_dev, readsize=32, progsize=32, lookahead=32)
Create a filesystem object that uses the `littlefs v1 filesystem format`_.
Storage of the littlefs filesystem is provided by *block_dev*, which must
@ -187,23 +187,31 @@ represented by VFS classes.
See :ref:`filesystem` for more information.
.. staticmethod:: mkfs(block_dev)
.. staticmethod:: mkfs(block_dev, readsize=32, progsize=32, lookahead=32)
Build a Lfs1 filesystem on *block_dev*.
.. note:: There are reports of littlefs v1 failing in certain situations,
for details see `littlefs issue 347`_.
.. class:: VfsLfs2(block_dev)
.. class:: VfsLfs2(block_dev, readsize=32, progsize=32, lookahead=32, mtime=True)
Create a filesystem object that uses the `littlefs v2 filesystem format`_.
Storage of the littlefs filesystem is provided by *block_dev*, which must
support the :ref:`extended interface <block-device-interface>`.
Objects created by this constructor can be mounted using :func:`mount`.
The *mtime* argument enables modification timestamps for files, stored using
littlefs attributes. This option can be disabled or enabled differently each
mount time and timestamps will only be added or updated if *mtime* is enabled,
otherwise the timestamps will remain untouched. Littlefs v2 filesystems without
timestamps will work without reformatting and timestamps will be added
transparently to existing files once they are opened for writing. When *mtime*
is enabled `uos.stat` on files without timestamps will return 0 for the timestamp.
See :ref:`filesystem` for more information.
.. staticmethod:: mkfs(block_dev)
.. staticmethod:: mkfs(block_dev, readsize=32, progsize=32, lookahead=32)
Build a Lfs2 filesystem on *block_dev*.

21
extmod/vfs_lfs.c
View File

@ -3,7 +3,7 @@
*
* The MIT License (MIT)
*
* Copyright (c) 2019 Damien P. George
* Copyright (c) 2019-2020 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -25,18 +25,20 @@
*/
#include "py/runtime.h"
#include "py/mphal.h"
#include "extmod/vfs.h"
#include "extmod/vfs_lfs.h"
#if MICROPY_VFS && (MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2)
enum { LFS_MAKE_ARG_bdev, LFS_MAKE_ARG_readsize, LFS_MAKE_ARG_progsize, LFS_MAKE_ARG_lookahead };
enum { LFS_MAKE_ARG_bdev, LFS_MAKE_ARG_readsize, LFS_MAKE_ARG_progsize, LFS_MAKE_ARG_lookahead, LFS_MAKE_ARG_mtime };
static const mp_arg_t lfs_make_allowed_args[] = {
{ MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ },
{ MP_QSTR_readsize, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 32} },
{ MP_QSTR_progsize, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 32} },
{ MP_QSTR_lookahead, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 32} },
{ MP_QSTR_mtime, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = true} },
};
#if MICROPY_VFS_LFS1
@ -98,9 +100,13 @@ mp_obj_t mp_vfs_lfs1_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode
#define MP_TYPE_VFS_LFSx mp_type_vfs_lfs2
#define MP_TYPE_VFS_LFSx_(s) mp_type_vfs_lfs2##s
// Attribute ids for lfs2_attr.type.
#define LFS_ATTR_MTIME (1) // 64-bit little endian, nanoseconds since 1970/1/1
typedef struct _mp_obj_vfs_lfs2_t {
mp_obj_base_t base;
mp_vfs_blockdev_t blockdev;
bool enable_mtime;
vstr_t cur_dir;
struct lfs2_config config;
lfs2_t lfs;
@ -109,14 +115,25 @@ typedef struct _mp_obj_vfs_lfs2_t {
typedef struct _mp_obj_vfs_lfs2_file_t {
mp_obj_base_t base;
mp_obj_vfs_lfs2_t *vfs;
uint8_t mtime[8];
lfs2_file_t file;
struct lfs2_file_config cfg;
struct lfs2_attr attrs[1];
uint8_t file_buffer[0];
} mp_obj_vfs_lfs2_file_t;
const char *mp_vfs_lfs2_make_path(mp_obj_vfs_lfs2_t *self, mp_obj_t path_in);
mp_obj_t mp_vfs_lfs2_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in);
STATIC void lfs_get_mtime(uint8_t buf[8]) {
uint64_t ns = mp_hal_time_ns();
// Store "ns" to "buf" in little-endian format (essentially htole64).
for (size_t i = 0; i < 8; ++i) {
buf[i] = ns;
ns >>= 8;
}
}
#include "extmod/vfs_lfsx.c"
#include "extmod/vfs_lfsx_file.c"

25
extmod/vfs_lfsx.c
View File

@ -3,7 +3,7 @@
*
* The MIT License (MIT)
*
* Copyright (c) 2019 Damien P. George
* Copyright (c) 2019-2020 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -34,6 +34,7 @@
#include "py/objstr.h"
#include "py/mperrno.h"
#include "extmod/vfs.h"
#include "lib/timeutils/timeutils.h"
STATIC int MP_VFS_LFSx(dev_ioctl)(const struct LFSx_API (config) * c, int cmd, int arg, bool must_return_int) {
mp_obj_t ret = mp_vfs_blockdev_ioctl(c->context, cmd, arg);
@ -120,6 +121,9 @@ STATIC mp_obj_t MP_VFS_LFSx(make_new)(const mp_obj_type_t * type, size_t n_args,
self->base.type = type;
vstr_init(&self->cur_dir, 16);
vstr_add_byte(&self->cur_dir, '/');
#if LFS_BUILD_VERSION == 2
self->enable_mtime = args[LFS_MAKE_ARG_mtime].u_bool;
#endif
MP_VFS_LFSx(init_config)(self, args[LFS_MAKE_ARG_bdev].u_obj,
args[LFS_MAKE_ARG_readsize].u_int, args[LFS_MAKE_ARG_progsize].u_int, args[LFS_MAKE_ARG_lookahead].u_int);
int ret = LFSx_API(mount)(&self->lfs, &self->config);
@ -352,6 +356,19 @@ STATIC mp_obj_t MP_VFS_LFSx(stat)(mp_obj_t self_in, mp_obj_t path_in) {
mp_raise_OSError(-ret);
}
mp_uint_t mtime = 0;
#if LFS_BUILD_VERSION == 2
uint8_t mtime_buf[8];
lfs2_ssize_t sz = lfs2_getattr(&self->lfs, path, LFS_ATTR_MTIME, &mtime_buf, sizeof(mtime_buf));
if (sz == sizeof(mtime_buf)) {
uint64_t ns = 0;
for (size_t i = sizeof(mtime_buf); i > 0; --i) {
ns = ns << 8 | mtime_buf[i - 1];
}
mtime = timeutils_seconds_since_2000_from_nanoseconds_since_1970(ns);
}
#endif
mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
t->items[0] = MP_OBJ_NEW_SMALL_INT(info.type == LFSx_MACRO(_TYPE_REG) ? MP_S_IFREG : MP_S_IFDIR); // st_mode
t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino
@ -360,9 +377,9 @@ STATIC mp_obj_t MP_VFS_LFSx(stat)(mp_obj_t self_in, mp_obj_t path_in) {
t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid
t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid
t->items[6] = mp_obj_new_int_from_uint(info.size); // st_size
t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime
t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime
t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime
t->items[7] = MP_OBJ_NEW_SMALL_INT(mtime); // st_atime
t->items[8] = MP_OBJ_NEW_SMALL_INT(mtime); // st_mtime
t->items[9] = MP_OBJ_NEW_SMALL_INT(mtime); // st_ctime
return MP_OBJ_FROM_PTR(t);
}

18
extmod/vfs_lfsx_file.c
View File

@ -3,7 +3,7 @@
*
* The MIT License (MIT)
*
* Copyright (c) 2019 Damien P. George
* Copyright (c) 2019-2020 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -101,6 +101,17 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod
#endif
o->cfg.buffer = &o->file_buffer[0];
#if LFS_BUILD_VERSION == 2
if (self->enable_mtime) {
lfs_get_mtime(&o->mtime[0]);
o->attrs[0].type = LFS_ATTR_MTIME;
o->attrs[0].buffer = &o->mtime[0];
o->attrs[0].size = sizeof(o->mtime);
o->cfg.attrs = &o->attrs[0];
o->cfg.attr_count = MP_ARRAY_SIZE(o->attrs);
}
#endif
const char *path = MP_VFS_LFSx(make_path)(self, path_in);
int ret = LFSx_API(file_opencfg)(&self->lfs, &o->file, path, flags, &o->cfg);
if (ret < 0) {
@ -131,6 +142,11 @@ STATIC mp_uint_t MP_VFS_LFSx(file_read)(mp_obj_t self_in, void *buf, mp_uint_t s
STATIC mp_uint_t MP_VFS_LFSx(file_write)(mp_obj_t self_in, const void *buf, mp_uint_t size, int *errcode) {
MP_OBJ_VFS_LFSx_FILE *self = MP_OBJ_TO_PTR(self_in);
MP_VFS_LFSx(check_open)(self);
#if LFS_BUILD_VERSION == 2
if (self->vfs->enable_mtime) {
lfs_get_mtime(&self->mtime[0]);
}
#endif
LFSx_API(ssize_t) sz = LFSx_API(file_write)(&self->vfs->lfs, &self->file, buf, size);
if (sz < 0) {
*errcode = -sz;

13
tests/extmod/vfs_lfs.py
View File

@ -35,6 +35,11 @@ class RAMBlockDevice:
return 0
def print_stat(st, print_size=True):
# don't print times (just check that they have the correct type)
print(st[:6], st[6] if print_size else -1, type(st[7]), type(st[8]), type(st[9]))
def test(bdev, vfs_class):
print("test", vfs_class)
@ -69,10 +74,10 @@ def test(bdev, vfs_class):
vfs.mkdir("testdir")
# stat a file
print(vfs.stat("test"))
print_stat(vfs.stat("test"))
# stat a dir (size seems to vary on LFS2 so don't print that)
print(vfs.stat("testdir")[:6])
print_stat(vfs.stat("testdir"), False)
# read
with vfs.open("test", "r") as f:
@ -112,8 +117,8 @@ def test(bdev, vfs_class):
# create file in directory to make sure paths are relative
vfs.open("test2", "w").close()
print(vfs.stat("test2"))
print(vfs.stat("/testdir/test2"))
print_stat(vfs.stat("test2"))
print_stat(vfs.stat("/testdir/test2"))
vfs.remove("test2")
# chdir back to root and remove testdir

16
tests/extmod/vfs_lfs.py.exp
View File

@ -7,8 +7,8 @@ test <class 'VfsLfs1'>
[('test', 32768, 0, 8), ('testdir', 16384, 0, 0)]
[]
[('test', 32768, 0, 8)]
(32768, 0, 0, 0, 0, 0, 8, 0, 0, 0)
(16384, 0, 0, 0, 0, 0)
(32768, 0, 0, 0, 0, 0) 8 <class 'int'> <class 'int'> <class 'int'>
(16384, 0, 0, 0, 0, 0) -1 <class 'int'> <class 'int'> <class 'int'>
littlefs
data length: 4096
write 0
@ -22,8 +22,8 @@ write 3
[('test', 32768, 0, 8), ('testdir', 16384, 0, 0)]
/
/testdir
(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
/
/testdir
/
@ -44,8 +44,8 @@ test <class 'VfsLfs2'>
[('testdir', 16384, 0, 0), ('test', 32768, 0, 8)]
[]
[('test', 32768, 0, 8)]
(32768, 0, 0, 0, 0, 0, 8, 0, 0, 0)
(16384, 0, 0, 0, 0, 0)
(32768, 0, 0, 0, 0, 0) 8 <class 'int'> <class 'int'> <class 'int'>
(16384, 0, 0, 0, 0, 0) -1 <class 'int'> <class 'int'> <class 'int'>
littlefs
data length: 4096
write 0
@ -59,8 +59,8 @@ write 3
[('test', 32768, 0, 8), ('testdir', 16384, 0, 0)]
/
/testdir
(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
/
/testdir
/

98
tests/extmod/vfs_lfs_mtime.py Normal file
View File

@ -0,0 +1,98 @@
# Test for VfsLfs using a RAM device, mtime feature
try:
import utime, uos
utime.sleep
uos.VfsLfs2
except (ImportError, AttributeError):
print("SKIP")
raise SystemExit
class RAMBlockDevice:
ERASE_BLOCK_SIZE = 1024
def __init__(self, blocks):
self.data = bytearray(blocks * self.ERASE_BLOCK_SIZE)
def readblocks(self, block, buf, off):
addr = block * self.ERASE_BLOCK_SIZE + off
for i in range(len(buf)):
buf[i] = self.data[addr + i]
def writeblocks(self, block, buf, off):
addr = block * self.ERASE_BLOCK_SIZE + off
for i in range(len(buf)):
self.data[addr + i] = buf[i]
def ioctl(self, op, arg):
if op == 4: # block count
return len(self.data) // self.ERASE_BLOCK_SIZE
if op == 5: # block size
return self.ERASE_BLOCK_SIZE
if op == 6: # erase block
return 0
def test(bdev, vfs_class):
print("test", vfs_class)
# Initial format of block device.
vfs_class.mkfs(bdev)
# construction
print("mtime=True")
vfs = vfs_class(bdev, mtime=True)
# Create an empty file, should have a timestamp.
vfs.open("test1", "wt").close()
# Wait 1 second so mtime will increase by at least 1.
utime.sleep(1)
# Create another empty file, should have a timestamp.
vfs.open("test2", "wt").close()
# Stat the files and check that test1 is older than test2.
stat1 = vfs.stat("test1")
stat2 = vfs.stat("test2")
print(stat1[8] != 0, stat2[8] != 0)
print(stat1[8] < stat2[8])
# Wait 1 second so mtime will increase by at least 1.
utime.sleep(1)
# Open test1 for reading and ensure mtime did not change.
vfs.open("test1", "rt").close()
print(vfs.stat("test1") == stat1)
# Open test1 for writing and ensure mtime increased from the previous value.
vfs.open("test1", "wt").close()
stat1_old = stat1
stat1 = vfs.stat("test1")
print(stat1_old[8] < stat1[8])
# Unmount.
vfs.umount()
# Check that remounting with mtime=False can read the timestamps.
print("mtime=False")
vfs = vfs_class(bdev, mtime=False)
print(vfs.stat("test1") == stat1)
print(vfs.stat("test2") == stat2)
f = vfs.open("test1", "wt")
f.close()
print(vfs.stat("test1") == stat1)
vfs.umount()
# Check that remounting with mtime=True still has the timestamps.
print("mtime=True")
vfs = vfs_class(bdev, mtime=True)
print(vfs.stat("test1") == stat1)
print(vfs.stat("test2") == stat2)
vfs.umount()
bdev = RAMBlockDevice(30)
test(bdev, uos.VfsLfs2)

13
tests/extmod/vfs_lfs_mtime.py.exp Normal file
View File

@ -0,0 +1,13 @@
test <class 'VfsLfs2'>
mtime=True
True True
True
True
True
mtime=False
True
True
True
mtime=True
True
True