Integrate dxcapi v2 and other changes from internal (#2575)

* Integrate changes from internal. - dxcapi v2 - new dxc options - DxilValueCache - PDB and NoOpt improvements - noop / llvm::donothing() support * Update dxrfallbacklayer for dxcapi internal changes * Reorder diag block based on whether pDiag is set first. * llvm::donothing() requires dxil 1.6 / SM 6.6 for now, lib as well. * Fixes for spir-v, non-VC compiler and non-Windows builds - DEFINE_CROSS_PLATFORM_UUIDOF for new interfaces - add SAL annotations - turn output argument validation for -P into warning - handle warnings without concatenating them to main output - update spirv preprocessing and compilation paths - return E_NOTIMPL from IDxcUtils::CreateReflection - cleanup: DxcContainerBuilder back to uft8, DxcTestUtils: remove comment * Fix some warnings from clang/gcc. * Fix unicode conversion problems on linux, where sizeof(wchar_t) == 4 Note this is an intermediate fix. On linux, what we are calling utf16 is actually a wide string that's probably utf32. This change fixes issues introduced by the new interface changes so things are consistent and pass tests. A future fix should correct the encodings so they are correctly labeled on platforms where wchar_t doesn't mean UTF16. * Return false for IsBufferNullTerminated when CP_ACP. One test for Disassembler was crashing because it created a pinned blob with a size of 1 << 31 + 1 without actual memory backing this. The IsBufferNullTerminated would attempt to see if this was null terminated, causing AV. This change also removes CP_UTF8 from this test when it was creating binary blobs, not UTF8 text blobs.
microsoft · Nov 14, 2019 · f4965b7 · f4965b7
1 parent 48d3b17
commit f4965b7
Show file tree

Hide file tree

Showing 128 changed files with 6,272 additions and 1,503 deletions.
diff --git a/include/dxc/DXIL/DxilMetadataHelper.h b/include/dxc/DXIL/DxilMetadataHelper.h
@@ -21,6 +21,7 @@ class LLVMContext;
 class Module;
 class Function;
 class Instruction;
+class DbgDeclareInst;
 class Value;
 class MDOperand;
 class Metadata;
@@ -54,6 +55,14 @@ struct DxilFunctionProps;
 class DxilSubobjects;
 class DxilSubobject;
 
+// Additional debug information for SROA'ed array variables,
+// where adjacent elements in DXIL might not have been adjacent
+// in the original user variable.
+struct DxilDIArrayDim {
+  unsigned StrideInBits;
+  unsigned NumElements;
+};
+
 /// Use this class to manipulate DXIL-spcific metadata.
 // In our code, only DxilModule and HLModule should use this class.
 class DxilMDHelper {
@@ -217,6 +226,9 @@ class DxilMDHelper {
   // NonUniform attribute.
   static const char kDxilNonUniformAttributeMDName[];
 
+  // Variable debug layout metadata.
+  static const char kDxilVariableDebugLayoutMDName[];
+
   // Validator version.
   static const char kDxilValidatorVersionMDName[];
   // Validator version uses the same constants for fields as kDxilVersion*
@@ -484,6 +496,10 @@ class DxilMDHelper {
   static void MarkPrecise(llvm::Instruction *inst);
   static bool IsMarkedNonUniform(const llvm::Instruction *inst);
   static void MarkNonUniform(llvm::Instruction *inst);
+  static bool GetVariableDebugLayout(llvm::DbgDeclareInst *inst,
+    unsigned &StartOffsetInBits, std::vector<DxilDIArrayDim> &ArrayDims);
+  static void SetVariableDebugLayout(llvm::DbgDeclareInst *inst,
+    unsigned StartOffsetInBits, const std::vector<DxilDIArrayDim> &ArrayDims);
 
 private:
   llvm::LLVMContext &m_Ctx;

diff --git a/include/dxc/DXIL/DxilPDB.h b/include/dxc/DXIL/DxilPDB.h
@@ -19,6 +19,7 @@ struct IMalloc;
 namespace hlsl {
 namespace pdb {
 
+  HRESULT LoadDataFromStream(IMalloc *pMalloc, IStream *pIStream, IDxcBlob **ppHash, IDxcBlob **ppContainer);
   HRESULT LoadDataFromStream(IMalloc *pMalloc, IStream *pIStream, IDxcBlob **pOutContainer);
   HRESULT WriteDxilPDB(IMalloc *pMalloc, IDxcBlob *pContainer, llvm::ArrayRef<BYTE> HashData, IDxcBlob **ppOutBlob);
 }

diff --git a/include/dxc/DxilContainer/DxilContainer.h b/include/dxc/DxilContainer/DxilContainer.h
@@ -49,7 +49,7 @@ enum class DxilShaderHashFlags : uint32_t {
 typedef struct DxilShaderHash {
   uint32_t Flags; // DxilShaderHashFlags
   uint8_t Digest[DxilContainerHashSize];
-} DxcShaderHash;
+} DxilShaderHash;
 
 struct DxilContainerVersion {
   uint16_t Major;
@@ -410,6 +410,7 @@ enum class SerializeDxilFlags : uint32_t {
   DebugNameDependOnSource     = 1 << 2, // Make the debug name depend on source (and not just final module).
   StripReflectionFromDxilPart = 1 << 3, // Strip Reflection info from DXIL part.
   IncludeReflectionPart       = 1 << 4, // Include reflection in STAT part.
+  StripRootSignature          = 1 << 5, // Strip Root Signature from main shader container.
 };
 inline SerializeDxilFlags& operator |=(SerializeDxilFlags& l, const SerializeDxilFlags& r) {
   l = static_cast<SerializeDxilFlags>(static_cast<int>(l) | static_cast<int>(r));

diff --git a/include/dxc/DxilContainer/DxilContainerAssembler.h b/include/dxc/DxilContainer/DxilContainerAssembler.h
@@ -52,7 +52,8 @@ void SerializeDxilContainerForModule(hlsl::DxilModule *pModule,
                                      llvm::StringRef DebugName,
                                      SerializeDxilFlags Flags,
                                      DxilShaderHash *pShaderHashOut = nullptr,
-                                     AbstractMemoryStream *pReflectionStreamOut = nullptr);
+                                     AbstractMemoryStream *pReflectionStreamOut = nullptr,
+                                     AbstractMemoryStream *pRootSigStreamOut = nullptr);
 void SerializeDxilContainerForRootSignature(hlsl::RootSignatureHandle *pRootSigHandle,
                                      AbstractMemoryStream *pStream);
 

diff --git a/include/dxc/DxilContainer/DxilPipelineStateValidation.h b/include/dxc/DxilContainer/DxilPipelineStateValidation.h
@@ -443,7 +443,7 @@ class DxilPipelineStateValidation
   // returns true if no errors occurred.
   bool InitFromPSV0(const void* pBits, uint32_t size) {
     if(!(pBits != nullptr)) return false;
-    uint8_t* pCurBits = (uint8_t*)pBits;
+    uint8_t* pCurBits = (uint8_t*)const_cast<void*>(pBits);
     uint32_t minsize = sizeof(PSVRuntimeInfo0) + sizeof(uint32_t) * 2;
     if(!(size >= minsize)) return false;
     m_uPSVRuntimeInfoSize = *((const uint32_t*)pCurBits);
@@ -534,7 +534,7 @@ class DxilPipelineStateValidation
 
       // Input to Output dependencies
       for (unsigned i = 0; i < 4; i++) {
-        if (m_pPSVRuntimeInfo1->SigOutputVectors[i] > 0 && m_pPSVRuntimeInfo1->SigInputVectors > 0) {
+        if (!IsMS() && m_pPSVRuntimeInfo1->SigOutputVectors[i] > 0 && m_pPSVRuntimeInfo1->SigInputVectors > 0) {
           minsize += PSVComputeInputOutputTableSize(m_pPSVRuntimeInfo1->SigInputVectors, m_pPSVRuntimeInfo1->SigOutputVectors[i]);
           if (!(size >= minsize)) return false;
           m_pInputToOutputTable = (uint32_t*)pCurBits;

diff --git a/include/dxc/DxilContainer/DxilRuntimeReflection.h b/include/dxc/DxilContainer/DxilRuntimeReflection.h
@@ -137,6 +137,7 @@ class StringTableReader {
   const char *Get(uint32_t offset) const {
     _Analysis_assume_(offset < m_size && m_table &&
                       m_table[m_size - 1] == '\0');
+    (void)m_size; // avoid unused private warning if use above is ignored.
     return m_table + offset;
   }
 };
@@ -185,6 +186,7 @@ class RawBytesReader {
       : m_table(table), m_size(size) {}
   const void *Get(uint32_t offset) const {
     _Analysis_assume_(offset < m_size && m_table);
+    (void)m_size; // avoid unused private warning if use above is ignored.
     return (const void*)(((const char*)m_table) + offset);
   }
 };

diff --git a/include/dxc/DxilRootSignature/DxilRootSignature.h b/include/dxc/DxilRootSignature/DxilRootSignature.h
@@ -240,7 +240,7 @@ struct DxilDescriptorRange {
 };
 struct DxilRootDescriptorTable {
   uint32_t NumDescriptorRanges;
-  _Field_size_full_(NumDescriptorRanges)  const DxilDescriptorRange *pDescriptorRanges;
+  _Field_size_full_(NumDescriptorRanges)  DxilDescriptorRange *pDescriptorRanges;
 };
 struct DxilRootConstants {
   uint32_t ShaderRegister;
@@ -275,7 +275,7 @@ struct DxilDescriptorRange1 {
 };
 struct DxilRootDescriptorTable1 {
   uint32_t NumDescriptorRanges;
-  _Field_size_full_(NumDescriptorRanges)  const DxilDescriptorRange1 *pDescriptorRanges;
+  _Field_size_full_(NumDescriptorRanges)  DxilDescriptorRange1 *pDescriptorRanges;
 };
 struct DxilRootParameter1 {
   DxilRootParameterType ParameterType;

diff --git a/include/dxc/HLSL/DxilConvergentName.h b/include/dxc/HLSL/DxilConvergentName.h
@@ -11,5 +11,5 @@
 #pragma once
 
 namespace hlsl {
-  static char *kConvergentFunctionPrefix = "dxil.convergent.marker.";
+  static const char *kConvergentFunctionPrefix = "dxil.convergent.marker.";
 }
diff --git a/include/dxc/HLSL/DxilValueCache.h b/include/dxc/HLSL/DxilValueCache.h
@@ -0,0 +1,77 @@
+//===--------- DxilValueCache.cpp - Dxil Constant Value Cache ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
+
+namespace llvm {
+
+class Module;
+class DominatorTree;
+
+struct DxilValueCache : public ModulePass {
+  static char ID;
+
+  // Special Weak Value to Weak Value map.
+  struct WeakValueMap {
+    struct ValueVH : public CallbackVH {
+      ValueVH(Value *V) : CallbackVH(V) {}
+      void allUsesReplacedWith(Value *) override { setValPtr(nullptr); }
+    };
+    struct ValueEntry {
+      WeakVH Value;
+      ValueVH Self;
+      ValueEntry() : Value(nullptr), Self(nullptr) {}
+      inline void Set(llvm::Value *Key, llvm::Value *V) { Self = Key; Value = V; }
+      inline bool IsStale() const { return Self == nullptr; }
+    };
+    ValueMap<const Value *, ValueEntry> Map;
+    Value *Get(Value *V);
+    void Set(Value *Key, Value *V);
+    bool Seen(Value *v);
+    void SetSentinel(Value *V);
+    void dump() const;
+  private:
+    Value *GetSentinel(LLVMContext &Ctx);
+    std::unique_ptr<Value> Sentinel;
+  };
+
+private:
+
+  WeakValueMap ValueMap;
+
+  void MarkAlwaysReachable(BasicBlock *BB);
+  void MarkNeverReachable(BasicBlock *BB);
+  bool IsAlwaysReachable_(BasicBlock *BB);
+  bool IsNeverReachable_(BasicBlock *BB);
+  Value *OptionallyGetValue(Value *V);
+  Value *ProcessValue(Value *V, DominatorTree *DT);
+
+  Value *ProcessAndSimplify_PHI(Instruction *I, DominatorTree *DT);
+  Value *ProcessAndSimpilfy_Br(Instruction *I, DominatorTree *DT);
+  Value *SimplifyAndCacheResult(Instruction *I, DominatorTree *DT);
+
+public:
+
+  const char *getPassName() const override;
+  DxilValueCache();
+
+  bool runOnModule(Module &M) override { return false; } // Doesn't do anything by itself.
+  void dump() const;
+  Value *GetValue(Value *V, DominatorTree *DT=nullptr);
+  bool IsAlwaysReachable(BasicBlock *BB, DominatorTree *DT=nullptr);
+  bool IsNeverReachable(BasicBlock *BB, DominatorTree *DT=nullptr);
+};
+
+void initializeDxilValueCachePass(class llvm::PassRegistry &);
+ModulePass *createDxilValueCachePass();
+
+}
+
+
diff --git a/include/dxc/Support/DxcLangExtensionsHelper.h b/include/dxc/Support/DxcLangExtensionsHelper.h
@@ -174,9 +174,9 @@ class DxcLangExtensionsHelper : public DxcLangExtensionsHelperApply {
 
     // Define a  little function to convert encoded blob into a string.
     auto GetErrorAsString = [&name](const CComPtr<IDxcBlobEncoding> &pBlobString) -> std::string {
-      CComPtr<IDxcBlobEncoding> pUTF8BlobStr;
-      if (SUCCEEDED(hlsl::DxcGetBlobAsUtf8(pBlobString, &pUTF8BlobStr)))
-        return std::string(static_cast<char*>(pUTF8BlobStr->GetBufferPointer()), pUTF8BlobStr->GetBufferSize());
+      CComPtr<IDxcBlobUtf8> pUTF8BlobStr;
+      if (SUCCEEDED(hlsl::DxcGetBlobAsUtf8(pBlobString, DxcGetThreadMallocNoRef(), &pUTF8BlobStr)))
+        return std::string(pUTF8BlobStr->GetStringPointer(), pUTF8BlobStr->GetStringLength());
       else
         return std::string("invalid semantic define " + name);
     };

diff --git a/include/dxc/Support/FileIOHelper.h b/include/dxc/Support/FileIOHelper.h
@@ -20,6 +20,8 @@
 // Forward declarations.
 struct IDxcBlob;
 struct IDxcBlobEncoding;
+struct IDxcBlobUtf8;
+struct IDxcBlobUtf16;
 
 namespace hlsl {
 
@@ -131,6 +133,26 @@ void WriteBinaryFile(_In_z_ LPCWSTR pFileName,
 UINT32 DxcCodePageFromBytes(_In_count_(byteLen) const char *bytes,
                             size_t byteLen) throw();
 
+// More general create blob functions, used by other functions
+// Null pMalloc means use current thread malloc.
+// bPinned will point to existing memory without managing it;
+// bCopy will copy to heap; bPinned and bCopy are mutually exclusive.
+// If encodingKnown, UTF-8 or UTF-16, and null-termination possible,
+// an IDxcBlobUtf8 or IDxcBlobUtf16 will be constructed.
+// If text, it's best if size includes null terminator when not copying,
+// otherwise IDxcBlobUtf8 or IDxcBlobUtf16 will not be constructed.
+HRESULT DxcCreateBlob(
+    LPCVOID pPtr, SIZE_T size, bool bPinned, bool bCopy,
+    bool encodingKnown, UINT32 codePage,
+    IMalloc *pMalloc, IDxcBlobEncoding **ppBlobEncoding) throw();
+// Create from blob references original blob.
+// Pass nonzero for offset or length for sub-blob reference.
+HRESULT DxcCreateBlobEncodingFromBlob(
+    IDxcBlob *pFromBlob, UINT32 offset, UINT32 length,
+    bool encodingKnown, UINT32 codePage,
+    IMalloc *pMalloc, IDxcBlobEncoding **ppBlobEncoding) throw();
+
+// Load files
 HRESULT
 DxcCreateBlobFromFile(_In_opt_ IMalloc *pMalloc, LPCWSTR pFileName,
                       _In_opt_ UINT32 *pCodePage,
@@ -191,16 +213,11 @@ DxcCreateBlobWithEncodingOnMallocCopy(
   _In_ IMalloc *pIMalloc, _In_bytecount_(size) LPCVOID pText, UINT32 size, UINT32 codePage,
   _COM_Outptr_ IDxcBlobEncoding **pBlobEncoding) throw();
 
-HRESULT DxcGetBlobAsUtf8(_In_ IDxcBlob *pBlob,
-                         _COM_Outptr_ IDxcBlobEncoding **pBlobEncoding) throw();
-HRESULT
-DxcGetBlobAsUtf8NullTerm(
-    _In_ IDxcBlob *pBlob,
-    _COM_Outptr_ IDxcBlobEncoding **ppBlobEncoding) throw();
-
+HRESULT DxcGetBlobAsUtf8(_In_ IDxcBlob *pBlob, _In_ IMalloc *pMalloc,
+                         _COM_Outptr_ IDxcBlobUtf8 **pBlobEncoding) throw();
 HRESULT
 DxcGetBlobAsUtf16(_In_ IDxcBlob *pBlob, _In_ IMalloc *pMalloc,
-                  _COM_Outptr_ IDxcBlobEncoding **pBlobEncoding) throw();
+                  _COM_Outptr_ IDxcBlobUtf16 **pBlobEncoding) throw();
 
 bool IsBlobNullOrEmpty(_In_opt_ IDxcBlob *pBlob) throw();
 

diff --git a/include/dxc/Support/HLSLOptions.h b/include/dxc/Support/HLSLOptions.h
@@ -102,6 +102,9 @@ class DxcOpts {
   llvm::StringRef OutputHeader; // OPT_Fh
   llvm::StringRef OutputObject; // OPT_Fo
   llvm::StringRef OutputWarningsFile; // OPT_Fe
+  llvm::StringRef OutputReflectionFile; // OPT_Fre
+  llvm::StringRef OutputRootSigFile; // OPT_Frs
+  llvm::StringRef OutputShaderHashFile; // OPT_Fsh
   llvm::StringRef Preprocess; // OPT_P
   llvm::StringRef TargetProfile; // OPT_target_profile
   llvm::StringRef VariableName; // OPT_Vn
@@ -112,6 +115,7 @@ class DxcOpts {
   llvm::StringRef FloatDenormalMode; // OPT_denorm
   std::vector<std::string> Exports; // OPT_exports
   llvm::StringRef DefaultLinkage; // OPT_default_linkage
+  unsigned DefaultTextCodePage = DXC_CP_UTF8; // OPT_encoding
 
   bool AllResourcesBound = false; // OPT_all_resources_bound
   bool AstDump = false; // OPT_ast_dump
@@ -144,7 +148,7 @@ class DxcOpts {
   bool UseHexLiterals = false; // OPT_Lx
   bool UseInstructionByteOffsets = false; // OPT_No
   bool UseInstructionNumbers = false; // OPT_Ni
-  bool NotUseLegacyCBufLoad = false;  // OPT_not_use_legacy_cbuf_load
+  bool NotUseLegacyCBufLoad = false;  // OPT_no_legacy_cbuf_layout
   bool PackPrefixStable = false;  // OPT_pack_prefix_stable
   bool PackOptimized = false;  // OPT_pack_optimized
   bool DisplayIncludeProcess = false; // OPT__vi