tools: Add gen-cpydiff.py to generate docs differences.
This patch introduces the a small framework to track differences between uPy and CPython. The framework consists of: - A set of "tests" which test for an individual feature that differs between uPy and CPy. Each test is like a normal uPy test in the test suite, but has a special comment at the start with some meta-data: a category (eg syntax, core language), a human-readable description of the difference, a cause, and a workaround. Following the meta-data there is a short code snippet which demonstrates the difference. See tests/cpydiff directory for the initial set of tests. - A program (this patch) which runs all the tests (on uPy and CPy) and generates nicely-formated .rst documenting the differences. - Integration into the docs build so that everything is automatic, and the differences appear in a way that is easy for users to read/reference (see latter commits). The idea with using this new framework is: - When a new difference is found it's easy to write a short test for it, along with a description, and add it to the existing ones. It's also easy for contributors to submit tests for differences they find. - When something is no longer different the tool will give an error and difference can be removed (or promoted to a proper feature test).
This commit is contained in:
Rami Ali
Damien George
parent
86c7507233
commit
b7fa63c7ce
|
|
@ -0,0 +1,8 @@
|
|||
MicroPython Differences from CPython
|
||||
====================================
|
||||
|
||||
The operations listed in this section produce conflicting results in MicroPython when compared to standard Python.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
|
|
@ -0,0 +1,213 @@
|
|||
# This file is part of the MicroPython project, http://micropython.org/
|
||||
#
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Copyright (c) 2016 Rami Ali
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
""" gen-cpydiff generates documentation which outlines operations that differ between MicroPython
|
||||
and CPython. This script is called by the docs Makefile for html and Latex and may be run
|
||||
manually using the command make gen-cpydiff. """
|
||||
|
||||
import os
|
||||
import errno
|
||||
import subprocess
|
||||
import time
|
||||
import re
|
||||
from collections import namedtuple
|
||||
|
||||
TESTPATH = '../tests/cpydiff/'
|
||||
UPYPATH = '../unix/micropython'
|
||||
DOCPATH = '../docs/genrst/'
|
||||
INDEXTEMPLATE = '../docs/differences/index_template.txt'
|
||||
INDEX = 'index.rst'
|
||||
|
||||
HEADER = '.. This document was generated by tools/gen-cpydiff.py\n\n'
|
||||
UIMPORTLIST = {'struct', 'collections', 'json'}
|
||||
CLASSMAP = {'Core': 'Core Language', 'Types': 'Builtin Types'}
|
||||
INDEXPRIORITY = ['syntax', 'core_language', 'builtin_types', 'modules']
|
||||
RSTCHARS = ['=', '-', '~', '`', ':']
|
||||
SPLIT = '"""\n|categories: |description: |cause: |workaround: '
|
||||
TAB = ' '
|
||||
|
||||
Output = namedtuple('output', ['name', 'class_', 'desc', 'cause', 'workaround', 'code',
|
||||
'output_cpy', 'output_upy', 'status'])
|
||||
|
||||
def readfiles():
|
||||
""" Reads test files """
|
||||
tests = list(filter(lambda x: x.endswith('.py'), os.listdir(TESTPATH)))
|
||||
tests.sort()
|
||||
files = []
|
||||
|
||||
for test in tests:
|
||||
text = open(TESTPATH + test, 'r').read()
|
||||
|
||||
try:
|
||||
class_, desc, cause, workaround, code = [x.rstrip() for x in \
|
||||
list(filter(None, re.split(SPLIT, text)))]
|
||||
output = Output(test, class_, desc, cause, workaround, code, '', '', '')
|
||||
files.append(output)
|
||||
except IndexError:
|
||||
print('Incorrect format in file ' + TESTPATH + test)
|
||||
|
||||
return files
|
||||
|
||||
def uimports(code):
|
||||
""" converts CPython module names into MicroPython equivalents """
|
||||
for uimport in UIMPORTLIST:
|
||||
uimport = bytes(uimport, 'utf8')
|
||||
code = code.replace(uimport, b'u' + uimport)
|
||||
return code
|
||||
|
||||
def run_tests(tests):
|
||||
""" executes all tests """
|
||||
results = []
|
||||
for test in tests:
|
||||
with open(TESTPATH + test.name, 'rb') as f:
|
||||
input_cpy = f.read()
|
||||
input_upy = uimports(input_cpy)
|
||||
|
||||
process = subprocess.Popen('python', shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
output_cpy = [com.decode('utf8') for com in process.communicate(input_cpy)]
|
||||
|
||||
process = subprocess.Popen(UPYPATH, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
output_upy = [com.decode('utf8') for com in process.communicate(input_upy)]
|
||||
|
||||
if output_cpy[0] == output_upy[0] and output_cpy[1] == output_upy[1]:
|
||||
status = 'Supported'
|
||||
print('Supported operation!\nFile: ' + TESTPATH + test.name)
|
||||
else:
|
||||
status = 'Unsupported'
|
||||
|
||||
output = Output(test.name, test.class_, test.desc, test.cause,
|
||||
test.workaround, test.code, output_cpy, output_upy, status)
|
||||
results.append(output)
|
||||
|
||||
results.sort(key=lambda x: x.class_)
|
||||
return results
|
||||
|
||||
def indent(block, spaces):
|
||||
""" indents paragraphs of text for rst formatting """
|
||||
new_block = ''
|
||||
for line in block.split('\n'):
|
||||
new_block += spaces + line + '\n'
|
||||
return new_block
|
||||
|
||||
def gen_table(contents):
|
||||
""" creates a table given any set of columns """
|
||||
xlengths = []
|
||||
ylengths = []
|
||||
for column in contents:
|
||||
col_len = 0
|
||||
for entry in column:
|
||||
lines = entry.split('\n')
|
||||
for line in lines:
|
||||
col_len = max(len(line) + 2, col_len)
|
||||
xlengths.append(col_len)
|
||||
for i in range(len(contents[0])):
|
||||
ymax = 0
|
||||
for j in range(len(contents)):
|
||||
ymax = max(ymax, len(contents[j][i].split('\n')))
|
||||
ylengths.append(ymax)
|
||||
|
||||
table_divider = '+' + ''.join(['-' * i + '+' for i in xlengths]) + '\n'
|
||||
table = table_divider
|
||||
for i in range(len(ylengths)):
|
||||
row = [column[i] for column in contents]
|
||||
row = [entry + '\n' * (ylengths[i]-len(entry.split('\n'))) for entry in row]
|
||||
row = [entry.split('\n') for entry in row]
|
||||
for j in range(ylengths[i]):
|
||||
k = 0
|
||||
for entry in row:
|
||||
width = xlengths[k]
|
||||
table += ''.join(['| {:{}}'.format(entry[j], width - 1)])
|
||||
k += 1
|
||||
table += '|\n'
|
||||
table += table_divider
|
||||
return table + '\n'
|
||||
|
||||
def gen_rst(results):
|
||||
""" creates restructured text documents to display tests """
|
||||
|
||||
# make sure the destination directory exists
|
||||
try:
|
||||
os.mkdir(DOCPATH)
|
||||
except OSError as e:
|
||||
if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
|
||||
raise
|
||||
|
||||
toctree = []
|
||||
class_ = []
|
||||
for output in results:
|
||||
section = output.class_.split(',')
|
||||
for i in range(len(section)):
|
||||
section[i] = section[i].rstrip()
|
||||
if section[i] in CLASSMAP:
|
||||
section[i] = CLASSMAP[section[i]]
|
||||
if i >= len(class_) or section[i] != class_[i]:
|
||||
if i == 0:
|
||||
filename = section[i].replace(' ', '_').lower()
|
||||
rst = open(DOCPATH + filename + '.rst', 'w')
|
||||
rst.write(HEADER)
|
||||
rst.write(section[i] + '\n')
|
||||
rst.write(RSTCHARS[0] * len(section[i]))
|
||||
rst.write(time.strftime("\nGenerated %a %d %b %Y %X UTC\n\n", time.gmtime()))
|
||||
toctree.append(filename)
|
||||
else:
|
||||
rst.write(section[i] + '\n')
|
||||
rst.write(RSTCHARS[min(i, len(RSTCHARS)-1)] * len(section[i]))
|
||||
rst.write('\n\n')
|
||||
class_ = section
|
||||
rst.write('**' + output.desc + '**\n\n')
|
||||
if output.cause != 'Unknown':
|
||||
rst.write('**Cause:** ' + output.cause + '\n\n')
|
||||
if output.workaround != 'Unknown':
|
||||
rst.write('**Workaround:** ' + output.workaround + '\n\n')
|
||||
|
||||
rst.write('Sample code::\n\n' + indent(output.code, TAB) + '\n')
|
||||
output_cpy = indent(''.join(output.output_cpy[0:2]), TAB).rstrip()
|
||||
output_cpy = ('::\n\n' if output_cpy != '' else '') + output_cpy
|
||||
output_upy = indent(''.join(output.output_upy[0:2]), TAB).rstrip()
|
||||
output_upy = ('::\n\n' if output_upy != '' else '') + output_upy
|
||||
table = gen_table([['CPy output:', output_cpy], ['uPy output:', output_upy]])
|
||||
rst.write(table)
|
||||
|
||||
template = open(INDEXTEMPLATE, 'r')
|
||||
index = open(DOCPATH + INDEX, 'w')
|
||||
index.write(HEADER)
|
||||
index.write(template.read())
|
||||
for section in INDEXPRIORITY:
|
||||
if section in toctree:
|
||||
index.write(indent(section + '.rst', TAB))
|
||||
toctree.remove(section)
|
||||
for section in toctree:
|
||||
index.write(indent(section + '.rst', TAB))
|
||||
|
||||
def main():
|
||||
""" Main function """
|
||||
|
||||
# clear search path to make sure tests use only builtin modules
|
||||
os.environ['MICROPYPATH'] = ''
|
||||
|
||||
files = readfiles()
|
||||
results = run_tests(files)
|
||||
gen_rst(results)
|
||||
|
||||
main()
|
||||
Loading…
Reference in New Issue