djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: parallelize the creation of qstr.i.last

Browse Source 
Rather than simply invoking gcc in preprocessor mode with a list of files, use
a Python script with the (python3) ThreadPoolExecutor to invoke the
preprocessor in parallel.

The amount of concurrency is the number of system CPUs, not the makefile "-j"
parallelism setting, because there is no simple and correct way for a Python
program to correctly work together with make's idea of parallelism.

This reduces the build time of stm32f405 feather (a non-LTO build) from 16s to
12s on my 16-thread Ryzen machine.
This commit is contained in:
Jeff Epler 2020-10-11 15:19:55 -05:00
parent 70a94c8d2d
commit 607e4a905a
2 changed files with 39 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
py/genlast.py Normal file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env python3
import sys
from concurrent.futures import ThreadPoolExecutor
import multiprocessing
import threading
import subprocess
def checkoutput1(args):
info = subprocess.run(args, check=True, stdout=subprocess.PIPE, input='')
return info.stdout
idx1 = sys.argv.index('--')
idx2 = sys.argv.index('--', idx1+1)
check = sys.argv[1:idx1]
always = sys.argv[idx1+1:idx2]
command = sys.argv[idx2+1:]
output_lock = threading.Lock()
def preprocess(fn):
output = checkoutput1(command + [fn])
# Ensure our output doesn't interleave with others
# a threading.Lock is not a context manager object :(
try:
output_lock.acquire()
sys.stdout.buffer.write(output)
finally:
output_lock.release()
def maybe_preprocess(fn):
if subprocess.call(["grep", "-lqE", "(MP_QSTR|translate)", fn]) == 0:
preprocess(fn)
executor = ThreadPoolExecutor(max_workers=multiprocessing.cpu_count() + 1)
executor.map(maybe_preprocess, check)
executor.map(preprocess, always)
executor.shutdown()

2
py/mkrules.mk
View File

@ -78,7 +78,7 @@ $(OBJ): | $(HEADER_BUILD)/qstrdefs.enum.h $(HEADER_BUILD)/mpversion.h
# - else, process all source files ($^) [this covers "make -B" which can set $? to empty]
$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(SRC_QSTR_PREPROCESSOR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
$(STEPECHO) "GEN $@"
$(Q)grep -lE "(MP_QSTR|translate)" $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) | xargs $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(SRC_QSTR_PREPROCESSOR) >$(HEADER_BUILD)/qstr.i.last;
$(Q)$(PYTHON3) $(PY_SRC)/genlast.py $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) -- $(SRC_QSTR_PREPROCESSOR) -- $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) >$(HEADER_BUILD)/qstr.i.last;
$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last $(PY_SRC)/makeqstrdefs.py
$(STEPECHO) "GEN $@"