Draft

clang/include/clang/AST/DeclID.h

@@ -197,6 +197,8 @@ class LocalDeclID : public DeclIDBase {
 
   static LocalDeclID get(ASTReader &Reader, serialization::ModuleFile &MF,
                          DeclID ID);
+  static LocalDeclID get(ASTReader &Reader, serialization::ModuleFile &MF,
+                         unsigned ModuleFileIndex, unsigned LocalDeclID);
 
   LocalDeclID &operator++() {
     ++ID;

clang/include/clang/Serialization/ASTBitCodes.h

@@ -261,6 +261,9 @@ using unaligned_decl_id_t =
         serialization::DeclID, llvm::endianness::native,
         llvm::support::unaligned>;
 
+/// The number of slots needed to record a DeclID in bitstreams.
+const unsigned int DeclIDRefSize = 2;
+
 /// The number of predefined preprocessed entity IDs.
 const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1;
 

clang/include/clang/Serialization/ASTReader.h

@@ -600,7 +600,7 @@ class ASTReader
 
   /// An array of lexical contents of a declaration context, as a sequence of
   /// Decl::Kind, DeclID pairs.
-  using LexicalContents = ArrayRef<serialization::unaligned_decl_id_t>;
+  using LexicalContents = ArrayRef<uint32_t>;
 
   /// Map from a DeclContext to its lexical contents.
   llvm::DenseMap<const DeclContext*, std::pair<ModuleFile*, LexicalContents>>
@@ -961,8 +961,7 @@ class ASTReader
   SmallVector<uint64_t, 8> DelayedDeleteExprs;
 
   // A list of late parsed template function data with their module files.
-  SmallVector<std::pair<ModuleFile *, SmallVector<uint64_t, 1>>, 4>
-      LateParsedTemplates;
+  SmallVector<std::pair<ModuleFile *, RecordData>, 4> LateParsedTemplates;
 
   /// The IDs of all decls to be checked for deferred diags.
   ///

clang/include/clang/Serialization/ASTRecordReader.h

@@ -187,12 +187,26 @@ class ASTRecordReader
   /// Reads a declaration from the given position in a record in the
   /// given module, advancing Idx.
   Decl *readDecl() {
+#ifndef NDEBUG
+    unsigned OldIdx = Idx;
+    Decl *D = Reader->ReadDecl(*F, Record, Idx);
+    assert(Idx - OldIdx == serialization::DeclIDRefSize);
+    return D;
+#endif
     return Reader->ReadDecl(*F, Record, Idx);
   }
   Decl *readDeclRef() {
     return readDecl();
   }
 
+  template <class DeclKind, class Func> void readDeclArray(Func &&ConsumeFunc) {
+    unsigned LengthOfArray = readInt();
+    unsigned End = Idx + LengthOfArray;
+
+    while (Idx < End)
+      ConsumeFunc(readDeclAs<DeclKind>());
+  }
+
   /// Reads a declaration from the given position in the record,
   /// advancing Idx.
   ///

clang/include/clang/Serialization/ASTRecordWriter.h

@@ -234,12 +234,39 @@ class ASTRecordWriter
 
   /// Emit a reference to a declaration.
   void AddDeclRef(const Decl *D) {
+#ifndef NDEBUG
+    unsigned OldSize = size();
+    Writer->AddDeclRef(D, *Record);
+    assert(size() - OldSize == serialization::DeclIDRefSize);
+    return;
+#endif
     return Writer->AddDeclRef(D, *Record);
   }
   void writeDeclRef(const Decl *D) {
     AddDeclRef(D);
   }
 
+  void writeNullDeclRef() {
+#ifndef NDEBUG
+    unsigned OldSize = size();
+#endif
+
+    push_back(0);
+    push_back(0);
+
+#ifndef NDEBUG
+    assert(size() - OldSize == serialization::DeclIDRefSize);
+#endif
+  }
+
+  template <class DeclKind> void writeDeclArray(ArrayRef<DeclKind *> Array) {
+    unsigned ElementNum = Array.size();
+    push_back(ElementNum * serialization::DeclIDRefSize);
+
+    for (DeclKind *D : Array)
+      AddDeclRef(D);
+  }
+
   /// Emit a declaration name.
   void AddDeclarationName(DeclarationName Name) {
     writeDeclarationName(Name);

clang/lib/Serialization/ASTReader.cpp

@@ -935,9 +935,8 @@ LocalDeclID LocalDeclID::get(ASTReader &Reader, ModuleFile &MF, DeclID Value) {
   return ID;
 }
 
-static LocalDeclID getLocalDeclID(ASTReader &Reader, ModuleFile &MF,
-                                  unsigned ModuleFileIndex,
-                                  unsigned LocalDeclID) {
+LocalDeclID LocalDeclID::get(ASTReader &Reader, ModuleFile &MF,
+                             unsigned ModuleFileIndex, unsigned LocalDeclID) {
   DeclID Value = (DeclID)ModuleFileIndex << 32 | (DeclID)LocalDeclID;
   return LocalDeclID::get(Reader, MF, Value);
 }
@@ -1303,9 +1302,8 @@ bool ASTReader::ReadLexicalDeclContextStorage(ModuleFile &M,
   auto &Lex = LexicalDecls[DC];
   if (!Lex.first) {
     Lex = std::make_pair(
-        &M, llvm::ArrayRef(
-                reinterpret_cast<const unaligned_decl_id_t *>(Blob.data()),
-                Blob.size() / sizeof(DeclID)));
+        &M, llvm::ArrayRef(reinterpret_cast<const uint32_t *>(Blob.data()),
+                           Blob.size() / sizeof(uint32_t)));
   }
   DC->setHasExternalLexicalStorage(true);
   return false;
@@ -3426,8 +3424,8 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
     case TU_UPDATE_LEXICAL: {
       DeclContext *TU = ContextObj->getTranslationUnitDecl();
       LexicalContents Contents(
-          reinterpret_cast<const unaligned_decl_id_t *>(Blob.data()),
-          static_cast<unsigned int>(Blob.size() / sizeof(DeclID)));
+          reinterpret_cast<const uint32_t *>(Blob.data()),
+          static_cast<unsigned int>(Blob.size() / sizeof(uint32_t)));
       TULexicalDecls.push_back(std::make_pair(&F, Contents));
       TU->setHasExternalLexicalStorage(true);
       break;
@@ -3713,16 +3711,16 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       break;
 
     case VTABLE_USES:
-      if (Record.size() % 3 != 0)
-        return llvm::createStringError(std::errc::illegal_byte_sequence,
-                                       "Invalid VTABLE_USES record");
-
       // Later tables overwrite earlier ones.
       // FIXME: Modules will have some trouble with this. This is clearly not
       // the right way to do this.
       VTableUses.clear();
 
       for (unsigned Idx = 0, N = Record.size(); Idx != N; /* In loop */) {
+        if (Idx > N)
+          return llvm::createStringError(std::errc::illegal_byte_sequence,
+                                         "Invalid VTABLE_USES record");
+
         VTableUses.push_back(
             {ReadDeclID(F, Record, Idx),
              ReadSourceLocation(F, Record, Idx).getRawEncoding(),
@@ -3731,21 +3729,20 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       break;
 
     case PENDING_IMPLICIT_INSTANTIATIONS:
-
-      if (Record.size() % 2 != 0)
-        return llvm::createStringError(
-            std::errc::illegal_byte_sequence,
-            "Invalid PENDING_IMPLICIT_INSTANTIATIONS block");
-
       for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
+        if (I > N)
+          return llvm::createStringError(
+              std::errc::illegal_byte_sequence,
+              "Invalid PENDING_IMPLICIT_INSTANTIATIONS block");
+
         PendingInstantiations.push_back(
             {ReadDeclID(F, Record, I),
              ReadSourceLocation(F, Record, I).getRawEncoding()});
       }
       break;
 
     case SEMA_DECL_REFS:
-      if (Record.size() != 3)
+      if (Record.size() != 3 * serialization::DeclIDRefSize)
         return llvm::createStringError(std::errc::illegal_byte_sequence,
                                        "Invalid SEMA_DECL_REFS block");
       for (unsigned I = 0, N = Record.size(); I != N; /*in loop*/)
@@ -3803,11 +3800,11 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
     }
 
     case DECL_UPDATE_OFFSETS:
-      if (Record.size() % 2 != 0)
-        return llvm::createStringError(
-            std::errc::illegal_byte_sequence,
-            "invalid DECL_UPDATE_OFFSETS block in AST file");
       for (unsigned I = 0, N = Record.size(); I != N; /*in loop*/) {
+        if (I > N)
+          return llvm::createStringError(
+              std::errc::illegal_byte_sequence,
+              "invalid DECL_UPDATE_OFFSETS block in AST file");
         GlobalDeclID ID = ReadDeclID(F, Record, I);
         DeclUpdateOffsets[ID].push_back(std::make_pair(&F, Record[I++]));
 
@@ -3820,12 +3817,12 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       break;
 
     case DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD: {
-      if (Record.size() % 3 != 0)
-        return llvm::createStringError(
-            std::errc::illegal_byte_sequence,
-            "invalid DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD block in AST "
-            "file");
       for (unsigned I = 0, N = Record.size(); I != N; /*in loop*/) {
+        if (I > N)
+          return llvm::createStringError(
+              std::errc::illegal_byte_sequence,
+              "invalid DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD block in AST "
+              "file");
         GlobalDeclID ID = ReadDeclID(F, Record, I);
 
         uint64_t BaseOffset = F.DeclsBlockStartOffset;
@@ -3915,10 +3912,11 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       break;
 
     case UNDEFINED_BUT_USED:
-      if (Record.size() % 2 != 0)
-        return llvm::createStringError(std::errc::illegal_byte_sequence,
-                                       "invalid undefined-but-used record");
       for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
+        if (I > N)
+          return llvm::createStringError(std::errc::illegal_byte_sequence,
+                                         "invalid undefined-but-used record");
+
         UndefinedButUsed.push_back(
             {ReadDeclID(F, Record, I),
              ReadSourceLocation(F, Record, I).getRawEncoding()});
@@ -7880,7 +7878,7 @@ LocalDeclID ASTReader::mapGlobalIDToModuleFileGlobalID(ModuleFile &M,
   if (!OrignalModuleFileIndex)
     return LocalDeclID();
 
-  return getLocalDeclID(*this, M, OrignalModuleFileIndex, ID);
+  return LocalDeclID::get(*this, M, OrignalModuleFileIndex, ID);
 }
 
 GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordDataImpl &Record,
@@ -7890,7 +7888,10 @@ GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordDataImpl &Record,
     return GlobalDeclID(0);
   }
 
-  return getGlobalDeclID(F, LocalDeclID::get(*this, F, Record[Idx++]));
+  uint32_t ModuleFileIndex = Record[Idx++];
+  uint32_t LocalDeclIndex = Record[Idx++];
+  return getGlobalDeclID(
+      F, LocalDeclID::get(*this, F, ModuleFileIndex, LocalDeclIndex));
 }
 
 /// Resolve the offset of a statement into a statement.
@@ -7919,25 +7920,26 @@ void ASTReader::FindExternalLexicalDecls(
     SmallVectorImpl<Decl *> &Decls) {
   bool PredefsVisited[NUM_PREDEF_DECL_IDS] = {};
 
-  auto Visit = [&] (ModuleFile *M, LexicalContents LexicalDecls) {
-    assert(LexicalDecls.size() % 2 == 0 && "expected an even number of entries");
-    for (int I = 0, N = LexicalDecls.size(); I != N; I += 2) {
+  auto Visit = [&](ModuleFile *M, LexicalContents LexicalDecls) {
+    assert(LexicalDecls.size() % 3 == 0 && "incorrect number of entries");
+    for (int I = 0, N = LexicalDecls.size(); I != N; I += 3) {
       auto K = (Decl::Kind)+LexicalDecls[I];
       if (!IsKindWeWant(K))
         continue;
 
-      auto ID = (DeclID) + LexicalDecls[I + 1];
+      LocalDeclID ID =
+          LocalDeclID::get(*this, *M, LexicalDecls[I + 1], LexicalDecls[I + 2]);
 
       // Don't add predefined declarations to the lexical context more
       // than once.
       if (ID < NUM_PREDEF_DECL_IDS) {
-        if (PredefsVisited[ID])
+        if (PredefsVisited[ID.getRawValue()])
           continue;
 
-        PredefsVisited[ID] = true;
+        PredefsVisited[ID.getRawValue()] = true;
       }
 
-      if (Decl *D = GetLocalDecl(*M, LocalDeclID::get(*this, *M, ID))) {
+      if (Decl *D = GetLocalDecl(*M, ID)) {
         assert(D->getKind() == K && "wrong kind for lexical decl");
         if (!DC->isDeclInLexicalTraversal(D))
           Decls.push_back(D);
@@ -8836,7 +8838,7 @@ void ASTReader::ReadLateParsedTemplates(
         &LPTMap) {
   for (auto &LPT : LateParsedTemplates) {
     ModuleFile *FMod = LPT.first;
-    RecordDataImpl &LateParsed = LPT.second;
+    const RecordData &LateParsed = LPT.second;
     for (unsigned Idx = 0, N = LateParsed.size(); Idx < N;
          /* In loop */) {
       FunctionDecl *FD = ReadDeclAs<FunctionDecl>(*FMod, LateParsed, Idx);

clang/lib/Serialization/ASTReaderDecl.cpp

@@ -1020,9 +1020,8 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   case FunctionDecl::TK_DependentFunctionTemplateSpecialization: {
     // Templates.
     UnresolvedSet<8> Candidates;
-    unsigned NumCandidates = Record.readInt();
-    while (NumCandidates--)
-      Candidates.addDecl(readDeclAs<NamedDecl>());
+    Record.readDeclArray<NamedDecl>(
+        [&Candidates](NamedDecl *ND) { Candidates.addDecl(ND); });
 
     // Templates args.
     TemplateArgumentListInfo TemplArgsWritten;
@@ -1152,11 +1151,9 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   FD->setIsPureVirtual(Pure);
 
   // Read in the parameters.
-  unsigned NumParams = Record.readInt();
   SmallVector<ParmVarDecl *, 16> Params;
-  Params.reserve(NumParams);
-  for (unsigned I = 0; I != NumParams; ++I)
-    Params.push_back(readDeclAs<ParmVarDecl>());
+  Record.readDeclArray<ParmVarDecl>(
+      [&Params](ParmVarDecl *ParmD) { Params.push_back(ParmD); });
   FD->setParams(Reader.getContext(), Params);
 }
 
@@ -2309,7 +2306,7 @@ void ASTDeclReader::VisitCXXMethodDecl(CXXMethodDecl *D) {
   } else {
     // We don't care about which declarations this used to override; we get
     // the relevant information from the canonical declaration.
-    Record.skipInts(NumOverridenMethods);
+    Record.skipInts(NumOverridenMethods * serialization::DeclIDRefSize);
   }
 }
 
@@ -4354,8 +4351,9 @@ void ASTReader::loadPendingDeclChain(Decl *FirstLocal, uint64_t LocalOffset) {
   // FIXME: We have several different dispatches on decl kind here; maybe
   // we should instead generate one loop per kind and dispatch up-front?
   Decl *MostRecent = FirstLocal;
-  for (unsigned I = 0, N = Record.size(); I != N; ++I) {
-    unsigned Idx = N - I - 1;
+  for (unsigned I = 0, N = Record.size(); I != N;
+       I += serialization::DeclIDRefSize) {
+    unsigned Idx = N - I - serialization::DeclIDRefSize;
     auto *D = ReadDecl(*M, Record, Idx);
     ASTDeclReader::attachPreviousDecl(*this, D, MostRecent, CanonDecl);
     MostRecent = D;

clang/lib/Serialization/ASTReaderStmt.cpp

@@ -351,12 +351,14 @@ void ASTStmtReader::VisitDeclStmt(DeclStmt *S) {
   S->setStartLoc(readSourceLocation());
   S->setEndLoc(readSourceLocation());
 
-  if (Record.size() - Record.getIdx() == 1) {
+  unsigned NumDecls =
+      (Record.size() - Record.getIdx()) / serialization::DeclIDRefSize;
+  if (NumDecls == 1) {
     // Single declaration
     S->setDeclGroup(DeclGroupRef(readDecl()));
   } else {
     SmallVector<Decl *, 16> Decls;
-    int N = Record.size() - Record.getIdx();
+    int N = NumDecls;
     Decls.reserve(N);
     for (int I = 0; I < N; ++I)
       Decls.push_back(readDecl());