circuitpython/ports/windows/msvc/genhdr.targets
Jim Mussared b326edf68c all: Remove MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE.
This commit removes all parts of code associated with the existing
MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE optimisation option, including the
-mcache-lookup-bc option to mpy-cross.

This feature originally provided a significant performance boost for Unix,
but wasn't able to be enabled for MCU targets (due to frozen bytecode), and
added significant extra complexity to generating and distributing .mpy
files.

The equivalent performance gain is now provided by the combination of
MICROPY_OPT_LOAD_ATTR_FAST_PATH and MICROPY_OPT_MAP_LOOKUP_CACHE (which has
been enabled on the unix port in the previous commit).

It's hard to provide precise performance numbers, but tests have been run
on a wide variety of architectures (x86-64, ARM Cortex, Aarch64, RISC-V,
xtensa) and they all generally agree on the qualitative improvements seen
by the combination of MICROPY_OPT_LOAD_ATTR_FAST_PATH and
MICROPY_OPT_MAP_LOOKUP_CACHE.

For example, on a "quiet" Linux x64 environment (i3-5010U @ 2.10GHz) the
change from CACHE_MAP_LOOKUP_IN_BYTECODE, to LOAD_ATTR_FAST_PATH combined
with MAP_LOOKUP_CACHE is:

diff of scores (higher is better)
N=2000 M=2000       bccache -> attrmapcache      diff      diff% (error%)
bm_chaos.py        13742.56 ->   13905.67 :   +163.11 =  +1.187% (+/-3.75%)
bm_fannkuch.py        60.13 ->      61.34 :     +1.21 =  +2.012% (+/-2.11%)
bm_fft.py         113083.20 ->  114793.68 :  +1710.48 =  +1.513% (+/-1.57%)
bm_float.py       256552.80 ->  243908.29 : -12644.51 =  -4.929% (+/-1.90%)
bm_hexiom.py         521.93 ->     625.41 :   +103.48 = +19.826% (+/-0.40%)
bm_nqueens.py     197544.25 ->  217713.12 : +20168.87 = +10.210% (+/-3.01%)
bm_pidigits.py      8072.98 ->    8198.75 :   +125.77 =  +1.558% (+/-3.22%)
misc_aes.py        17283.45 ->   16480.52 :   -802.93 =  -4.646% (+/-0.82%)
misc_mandel.py     99083.99 ->  128939.84 : +29855.85 = +30.132% (+/-5.88%)
misc_pystone.py    83860.10 ->   82592.56 :  -1267.54 =  -1.511% (+/-2.27%)
misc_raytrace.py   21490.40 ->   22227.23 :   +736.83 =  +3.429% (+/-1.88%)

This shows that the new optimisations are at least as good as the existing
inline-bytecode-caching, and are sometimes much better (because the new
ones apply caching to a wider variety of map lookups).

The new optimisations can also benefit code generated by the native
emitter, because they apply to the runtime rather than the generated code.
The improvement for the native emitter when LOAD_ATTR_FAST_PATH and
MAP_LOOKUP_CACHE are enabled is (same Linux environment as above):

diff of scores (higher is better)
N=2000 M=2000        native -> nat-attrmapcache  diff      diff% (error%)
bm_chaos.py        14130.62 ->   15464.68 :  +1334.06 =  +9.441% (+/-7.11%)
bm_fannkuch.py        74.96 ->      76.16 :     +1.20 =  +1.601% (+/-1.80%)
bm_fft.py         166682.99 ->  168221.86 :  +1538.87 =  +0.923% (+/-4.20%)
bm_float.py       233415.23 ->  265524.90 : +32109.67 = +13.756% (+/-2.57%)
bm_hexiom.py         628.59 ->     734.17 :   +105.58 = +16.796% (+/-1.39%)
bm_nqueens.py     225418.44 ->  232926.45 :  +7508.01 =  +3.331% (+/-3.10%)
bm_pidigits.py      6322.00 ->    6379.52 :    +57.52 =  +0.910% (+/-5.62%)
misc_aes.py        20670.10 ->   27223.18 :  +6553.08 = +31.703% (+/-1.56%)
misc_mandel.py    138221.11 ->  152014.01 : +13792.90 =  +9.979% (+/-2.46%)
misc_pystone.py    85032.14 ->  105681.44 : +20649.30 = +24.284% (+/-2.25%)
misc_raytrace.py   19800.01 ->   23350.73 :  +3550.72 = +17.933% (+/-2.79%)

In summary, compared to MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE, the new
MICROPY_OPT_LOAD_ATTR_FAST_PATH and MICROPY_OPT_MAP_LOOKUP_CACHE options:
- are simpler;
- take less code size;
- are faster (generally);
- work with code generated by the native emitter;
- can be used on embedded targets with a small and constant RAM overhead;
- allow the same .mpy bytecode to run on all targets.

See #7680 for further discussion.  And see also #7653 for a discussion
about simplifying mpy-cross options.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
2021-09-16 16:04:03 +10:00

146 lines
7.9 KiB
XML

<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="GenerateHeaders">
<Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
<!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
</Target>
<PropertyGroup>
<DestDir>$(PyBuildDir)genhdr\</DestDir>
<PySrcDir>$(PyBaseDir)py\</PySrcDir>
<QstrDefs>$(PyBaseDir)ports\unix\qstrdefsport.h</QstrDefs>
<PyQstrDefs>$(PySrcDir)qstrdefs.h</PyQstrDefs>
<QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
<QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
<PyPython Condition="'$(PyPython)' == ''">$(MICROPY_CPYTHON3)</PyPython>
<PyPython Condition="'$(PyPython)' == ''">python</PyPython>
<CLToolExe Condition="'$(CLToolExe)' == ''">cl.exe</CLToolExe>
<PyClTool>$([System.IO.Path]::Combine(`$(CLToolPath)`, `$(CLToolExe)`))</PyClTool>
</PropertyGroup>
<Target Name="MakeDestDir">
<MakeDir Directories="$(DestDir)"/>
</Target>
<!-- Concatenate preprocessed files for use with makeqstrdefs.py.
Filters out any lines which aren't used by makeqstrdefs.py so the resulting file is
hundreds of times smaller and processing in python takes substantially less time-->
<UsingTask TaskName="ConcatPreProcFiles" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll" >
<ParameterGroup>
<OutputFile Required="true" ParameterType="System.String"/>
<InputFiles Required="true" ParameterType="System.String[]"/>
</ParameterGroup>
<Task>
<Code Type="Fragment" Language="cs">
<![CDATA[
using(var outFile = System.IO.File.CreateText(OutputFile)) {
foreach(var inFile in InputFiles)
foreach(var line in System.IO.File.ReadAllLines(inFile))
if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
outFile.WriteLine( line );
}
]]>
</Code>
</Task>
</UsingTask>
<!-- If any of these changed we'll force all qstrs to be generated again-->
<ItemGroup>
<QstrDependencies Include="$(PyWinDir)mpconfigport.h;$(PySrcDir)mpconfig.h;$(PySrcDir)makeqstrdata.py"/>
</ItemGroup>
<!-- Preprocess changed files, concatenate and feed into makeqstrdefs.py split/cat-->
<Target Name="MakeQstrDefs" DependsOnTargets="MakeDestDir" Inputs="@(ClCompile);@(QstrDependencies)" Outputs="$(QstrDefsCollected)">
<ItemGroup>
<PyIncDirs Include="$(PyIncDirs)"/>
<PreProcDefs Include="%(ClCompile.PreProcessorDefinitions);NO_QSTR"/>
<PyQstrSourceFiles Include="@(ClCompile)" Exclude="$(PyBuildDir)\frozen_content.c">
<OutFile>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)qstr\'))</OutFile>
</PyQstrSourceFiles>
<PyQstrSourceFiles>
<OutFile>$([System.IO.Path]::ChangeExtension('%(OutFile)', '.pp'))</OutFile>
<OutDir>$([System.IO.Path]::GetDirectoryName('%(OutFile)'))</OutDir>
</PyQstrSourceFiles>
<PyQstrSourceFiles>
<Changed Condition="$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('%(OutFile)')))) &gt; 0">True</Changed>
</PyQstrSourceFiles>
<QstrDependencies>
<Changed Condition="$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('$(DestDir)qstr.i.last')))) &gt; 0">True</Changed>
</QstrDependencies>
</ItemGroup>
<PropertyGroup>
<ForceQstrRebuild>@(QstrDependencies->AnyHaveMetadataValue('Changed', 'True'))</ForceQstrRebuild>
<RunPreProcConcat>@(PyQstrSourceFiles->AnyHaveMetadataValue('Changed', 'True'))</RunPreProcConcat>
</PropertyGroup>
<MakeDir Directories="@(PyQstrSourceFiles->'%(OutDir)')"/>
<Exec Command="$(PyClTool) /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /Fi%(PyQstrSourceFiles.OutFile) /P %(PyQstrSourceFiles.Identity)"
Condition="'%(PyQstrSourceFiles.Changed)' == 'True' Or '$(ForceQstrRebuild)' == 'True'"/>
<ConcatPreProcFiles InputFiles="@(PyQstrSourceFiles->'%(OutFile)')" OutputFile="$(DestDir)qstr.i.last"
Condition="'$(RunPreProcConcat)' == 'True' Or '$(ForceQstrRebuild)' == 'True'"/>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split qstr $(DestDir)qstr.i.last $(DestDir)qstr _"/>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat qstr _ $(DestDir)qstr $(QstrDefsCollected)"/>
</Target>
<Target Name="MakeModuleDefs" DependsOnTargets="MakeDestDir">
<PropertyGroup>
<DestFile>$(DestDir)moduledefs.h</DestFile>
<TmpFile>$(DestFile).tmp</TmpFile>
</PropertyGroup>
<ItemGroup>
<PyUserModuleFiles Include="@(ClCompile)">
<Path>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', ''))</Path>
</PyUserModuleFiles>
</ItemGroup>
<Exec Command="$(PyPython) $(PySrcDir)makemoduledefs.py --vpath=&quot;., $(PyBaseDir), $(PyUserCModules)&quot; @(PyUserModuleFiles->'%(Path)', ' ') > $(TmpFile)"/>
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
</Target>
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
<PropertyGroup>
<TmpFile>$(QstrGen).tmp</TmpFile>
</PropertyGroup>
<Exec Command="$(PyClTool) /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /E $(PyQstrDefs) $(QstrDefs) > $(DestDir)qstrdefs.preprocessed.h"/>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdata.py $(DestDir)qstrdefs.preprocessed.h $(QstrDefsCollected) > $(TmpFile)"/>
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(QstrGen)"/>
</Target>
<Target Name="MakeVersionHdr" DependsOnTargets="MakeDestDir">
<PropertyGroup>
<DestFile>$(DestDir)mpversion.h</DestFile>
<TmpFile>$(DestFile).tmp</TmpFile>
</PropertyGroup>
<Exec Command="$(PyPython) $(PySrcDir)makeversionhdr.py $(TmpFile)"/>
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
</Target>
<Target Name="FreezeModules" Condition="'$(FrozenManifest)' != ''" DependsOnTargets="MakeQstrData" Inputs="$(FrozenManifest)" Outputs="$(PyBuildDir)frozen_content.c">
<ItemGroup>
<ClCompile Include="$(PyBuildDir)frozen_content.c"/>
<ClCompile>
<PreprocessorDefinitions>MICROPY_MODULE_FROZEN_MPY=1;MICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
</ItemGroup>
<Exec Command="$(PyPython) $(PyBaseDir)tools\makemanifest.py -v MPY_DIR=$(PyBaseDir) -v MPY_LIB_DIR=$(PyBaseDir)../micropython-lib -v PORT_DIR=$(PyWinDir) -o $(PyBuildDir)frozen_content.c -b $(PyBuildDir) $(FrozenManifest)"/>
<WriteLinesToFile File="$(TLogLocation)frozen.read.1.tlog" Lines="$(FrozenManifest)" Overwrite="True"/>
</Target>
<Target Name="RemoveGeneratedFiles" AfterTargets="Clean">
<RemoveDir Directories="$(DestDir)"/>
</Target>
<!--Copies SourceFile to DestFile only if SourceFile's content differs from DestFile's.
We use this to 'touch' the generated files only when they are really newer
so a build is only triggered if the generated content actually changed,
and not just because the file date changed since the last build-->
<Target Name="CopyFileIfDifferent">
<Exec Command="fc /B $(SourceFile) $(DestFile) > NUL 2>&amp;1" IgnoreExitCode="true">
<Output TaskParameter="ExitCode" PropertyName="FilesDiffer" />
</Exec>
<Copy SourceFiles="$(SourceFile)" DestinationFiles="$(DestFile)" Condition="'$(FilesDiffer)'!='0'"/>
</Target>
</Project>