From 60f5b941e096aba270a7d4af8faa33169de8364f Mon Sep 17 00:00:00 2001
From: Damien George <damien@micropython.org>
Date: Sat, 8 Aug 2020 15:39:56 +1000
Subject: [PATCH] extmod/vfs_reader: Fix mp_reader_new_file to open file in
 "rb" mode.

mp_reader_new_file() is used to read in files for importing, either .py or
.mpy files, for the lexer and persistent code loader respectively.  In both
cases the file should be opened in raw bytes mode: the lexer handles
unicode characters itself, and .mpy files contain 8-bit bytes by nature.

Before this commit importing was working correctly because, although the
file was opened in text mode, all native filesystem implementations (POSIX,
FAT, LFS) would access the file in raw bytes mode via mp_stream_rw()
calling mp_stream_p_t.read().  So it was only an issue for non-native
filesystems, such as those implemented in Python.  For Python-based
filesystem implementations, a call to mp_stream_rw() would go via IOBase
and then to readinto() at the Python level, and readinto() is only defined
on files opened in raw bytes mode.

Signed-off-by: Damien George <damien@micropython.org>
---
 extmod/vfs_reader.c            |  7 +++++--
 tests/extmod/vfs_userfs.py     | 14 ++++++++++----
 tests/extmod/vfs_userfs.py.exp |  6 +++---
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c
index db13ce3c33..d3904c5c50 100644
--- a/extmod/vfs_reader.c
+++ b/extmod/vfs_reader.c
@@ -71,8 +71,11 @@ STATIC void mp_reader_vfs_close(void *data) {
 
 void mp_reader_new_file(mp_reader_t *reader, const char *filename) {
     mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t);
-    mp_obj_t arg = mp_obj_new_str(filename, strlen(filename));
-    rf->file = mp_vfs_open(1, &arg, (mp_map_t *)&mp_const_empty_map);
+    mp_obj_t args[2] = {
+        mp_obj_new_str(filename, strlen(filename)),
+        MP_OBJ_NEW_QSTR(MP_QSTR_rb),
+    };
+    rf->file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
     int errcode;
     rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
     if (errcode != 0) {
diff --git a/tests/extmod/vfs_userfs.py b/tests/extmod/vfs_userfs.py
index 06e546b081..3cdfe82eea 100644
--- a/tests/extmod/vfs_userfs.py
+++ b/tests/extmod/vfs_userfs.py
@@ -16,14 +16,20 @@ except (ImportError, AttributeError):
 
 
 class UserFile(uio.IOBase):
-    def __init__(self, data):
+    def __init__(self, mode, data):
+        assert isinstance(data, bytes)
+        self.is_text = mode.find("b") == -1
         self.data = data
         self.pos = 0
 
     def read(self):
-        return self.data
+        if self.is_text:
+            return str(self.data, "utf8")
+        else:
+            return self.data
 
     def readinto(self, buf):
+        assert not self.is_text
         n = 0
         while n < len(buf) and self.pos < len(self.data):
             buf[n] = self.data[self.pos]
@@ -54,12 +60,12 @@ class UserFS:
 
     def open(self, path, mode):
         print("open", path, mode)
-        return UserFile(self.files[path])
+        return UserFile(mode, self.files[path])
 
 
 # create and mount a user filesystem
 user_files = {
-    "/data.txt": b"some data in a text file\n",
+    "/data.txt": b"some data in a text file",
     "/usermod1.py": b"print('in usermod1')\nimport usermod2",
     "/usermod2.py": b"print('in usermod2')",
 }
diff --git a/tests/extmod/vfs_userfs.py.exp b/tests/extmod/vfs_userfs.py.exp
index 6a4d925b91..00ddd95fca 100644
--- a/tests/extmod/vfs_userfs.py.exp
+++ b/tests/extmod/vfs_userfs.py.exp
@@ -1,12 +1,12 @@
 open /data.txt r
-b'some data in a text file\n'
+some data in a text file
 stat /usermod1
 stat /usermod1.py
-open /usermod1.py r
+open /usermod1.py rb
 ioctl 4 0
 in usermod1
 stat /usermod2
 stat /usermod2.py
-open /usermod2.py r
+open /usermod2.py rb
 ioctl 4 0
 in usermod2