djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

stmhal/make-stmconst.py: Add support for files with invalid utf8 bytes.

Browse Source
This commit is contained in:
Damien George 2016-11-16 23:29:02 +11:00
parent 4d9dce7759
commit c5621529c9
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
stmhal/make-stmconst.py
View File

@ -42,12 +42,17 @@ class Lexer:
) )
def __init__(self, filename): def __init__(self, filename):
self.file = open(filename, 'rt') self.file = open(filename, 'rb')
self.line_number = 0 self.line_number = 0
def next_match(self, strictly_next=False): def next_match(self, strictly_next=False):
while True: while True:
line = self.file.readline() line = self.file.readline()
try:
line = str(line, 'utf8')
except ValueError:
# some files have invalid utf8 bytes, so filter them out
line = ''.join(chr(l) for l in line if l <= 126)
self.line_number += 1 self.line_number += 1
if len(line) == 0: if len(line) == 0:
return ('EOF', None) return ('EOF', None)