djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

windows/msvc: Rewrite qstr auto-generation.

Browse Source 
Builds have been broken since reworking autogeneration in c618f91 and
related, this gets fixed here by applying similar qstr generation logic
for the msvc builds: c files are only preprocessed when changed (or not
yet preprocessed) and the concatenated output is fed into makeqstrdefs.py.
To speed up this process, the concatenated output is already filtered to
contain only lines which makeqstrdefs really needs: this makes the qstr
generation stage about twice as fast (checked on different machines).
This commit is contained in:
stijn 2016-04-23 18:55:14 +02:00 committed by Damien George
parent 9264d42e2a
commit 29c8c8aecb
1 changed files with 45 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
windows/msvc/genhdr.targets
View File

@ -3,7 +3,7 @@
<Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/> <Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
<!--Generate qstrdefs.h and mpversion.h similar to what is done in py/mkrules.mk--> <!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeQstrData"> <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeQstrData">
</Target> </Target>
@ -11,65 +11,70 @@
<DestDir>$(PyBuildDir)genhdr\</DestDir> <DestDir>$(PyBuildDir)genhdr\</DestDir>
<PySrcDir>$(PyBaseDir)py\</PySrcDir> <PySrcDir>$(PyBaseDir)py\</PySrcDir>
<QstrDefs>$(PyBaseDir)unix\qstrdefsport.h</QstrDefs> <QstrDefs>$(PyBaseDir)unix\qstrdefsport.h</QstrDefs>
<PyQstrDefs>$(PySrcDir)qstrdefs.h</PyQstrDefs>
<QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected> <QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
<QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen> <QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
<PyPython Condition="'$(PyPython)' == ''">python</PyPython> <PyPython Condition="'$(PyPython)' == ''">python</PyPython>
</PropertyGroup> </PropertyGroup>
<ItemGroup>
<PyQstrSourceFiles Include="@(ClCompile);$(PySrcDir)qstrdefs.h"/>
</ItemGroup>
<Target Name="MakeDestDir"> <Target Name="MakeDestDir">
<MakeDir Directories="$(DestDir)"/> <MakeDir Directories="$(DestDir)"/>
</Target> </Target>
<Target Name="MakeQstrDefs" DependsOnTargets="MakeDestDir" Inputs="@(PyQstrSourceFiles)" Outputs="$(QstrDefsCollected)"> <!-- Concatenate preprocessed files for use with makeqstrdefs.py.
Filters out any lines which aren't used by makeqstrdefs.py so the resulting file is
hundreds of times smaller and processing in python takes substantially less time-->
<UsingTask TaskName="ConcatPreProcFiles" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll" >
<ParameterGroup>
<OutputFile Required="true" ParameterType="System.String"/>
<InputFiles Required="true" ParameterType="System.String[]"/>
</ParameterGroup>
<Task>
<Code Type="Fragment" Language="cs">
<![CDATA[
using(var outFile = System.IO.File.CreateText(OutputFile)) {
foreach(var inFile in InputFiles)
foreach(var line in System.IO.File.ReadAllLines(inFile))
if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
outFile.WriteLine( line );
}
]]>
</Code>
</Task>
</UsingTask>
<!-- Preprocess changed files, concatenate and feed into makeqstrdefs.py split/cat-->
<Target Name="MakeQstrDefs" DependsOnTargets="MakeDestDir" Inputs="@(ClCompile)" Outputs="$(QstrDefsCollected)">
<ItemGroup> <ItemGroup>
<PyIncDirs Include="$(PyIncDirs)"/> <PyIncDirs Include="$(PyIncDirs)"/>
<PreProcDefs Include=" %(ClCompile.PreProcessorDefinitions)"/> <PreProcDefs Include="%(ClCompile.PreProcessorDefinitions);__QSTR_EXTRACT;N_X64;N_X86;N_THUMB;N_ARM"/>
<PyQstrSourceFiles> <PyQstrSourceFiles Include="@(ClCompile)">
<Qstr>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)'))</Qstr> <OutFile>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)qstr\'))</OutFile>
</PyQstrSourceFiles> </PyQstrSourceFiles>
<PyQstrSourceFiles> <PyQstrSourceFiles>
<Qstr>$([System.IO.Path]::ChangeExtension('%(Qstr)', '.qstr'))</Qstr> <OutFile>$([System.IO.Path]::ChangeExtension('%(OutFile)', '.pp'))</OutFile>
<PreProc>$([System.IO.Path]::ChangeExtension('%(Qstr)', '.pp'))</PreProc> <OutDir>$([System.IO.Path]::GetDirectoryName('%(OutFile)'))</OutDir>
<QstrDir>$([System.IO.Path]::GetDirectoryName('%(Qstr)'))</QstrDir> </PyQstrSourceFiles>
<PreProcOnly>$([System.String]::new('%(FileName)').Contains('qstrdefs'))</PreProcOnly> <PyQstrSourceFiles>
<Changed>$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('%(OutFile)'))))</Changed>
</PyQstrSourceFiles> </PyQstrSourceFiles>
</ItemGroup> </ItemGroup>
<PropertyGroup>
</PropertyGroup> <MakeDir Directories="@(PyQstrSourceFiles->'%(OutDir)')"/>
<Exec Command="cl /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /Fi%(PyQstrSourceFiles.OutFile) /P %(PyQstrSourceFiles.Identity)"
<!-- Preprocess and pass to makeqstrdefs if needed, else just copy --> Condition="%(PyQstrSourceFiles.Changed) &gt; 0"/>
<MakeDir Directories="@(PyQstrSourceFiles->'%(QstrDir)')"/> <ConcatPreProcFiles InputFiles="@(PyQstrSourceFiles->'%(OutFile)')" OutputFile="$(DestDir)qstr.i.last"
<Touch Files="$(QstrGen)" AlwaysCreate="true"/> Condition="%(PyQstrSourceFiles.Changed) &gt; 0"/>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
<Exec Command="cl /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /Fi%(PyQstrSourceFiles.PreProc) /P %(PyQstrSourceFiles.Identity)"/> <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py -s -o %(PyQstrSourceFiles.Qstr) %(PyQstrSourceFiles.PreProc)" Condition="'%(PyQstrSourceFiles.PreProcOnly)' != 'True'"/>
<Copy SourceFiles="%(PyQstrSourceFiles.PreProc)" DestinationFiles="%(PyQstrSourceFiles.Qstr)" Condition="'%(PyQstrSourceFiles.PreProcOnly)' == 'True'"/>
<!-- Collect all output (where qstrdefs file(s) have priority over autogenerated ones), then
filter out lines which definitely aren't qstr definitions so we don't end up with a huge (> 10mb) filesize -->
<ReadLinesFromFile File="%(PyQstrSourceFiles.Qstr)" Condition="'%(PyQstrSourceFiles.PreProcOnly)' == 'True'">
<Output TaskParameter="Lines" ItemName="PreProcLines"/>
</ReadLinesFromFile>
<ReadLinesFromFile File="%(PyQstrSourceFiles.Qstr)">
<Output TaskParameter="Lines" ItemName="PreProcLines"/>
</ReadLinesFromFile>
<ItemGroup>
<QStrLines Include="@(PreProcLines)" Condition="$([System.String]::new('%(Identity)').Contains('Q'))"/>
</ItemGroup>
<WriteLinesToFile Lines="@(QStrLines)" File="$(QstrDefsCollected)" Overwrite="true"/>
</Target> </Target>
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs"> <Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
<PropertyGroup> <PropertyGroup>
<TmpFile>$(DestFile).tmp</TmpFile> <TmpFile>$(QstrGen).tmp</TmpFile>
</PropertyGroup> </PropertyGroup>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdata.py $(QstrDefs) $(QstrDefsCollected) > $(TmpFile)"/> <Exec Command="cl /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /E $(PyQstrDefs) $(QstrDefs) > $(DestDir)qstrdefspreprocessed.h"/>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdata.py $(DestDir)qstrdefspreprocessed.h $(QstrDefsCollected) > $(TmpFile)"/>
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(QstrGen)"/> <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(QstrGen)"/>
</Target> </Target>