Skip to content

Commit

Permalink
Wrap various major convert tasks in signpost intervals
Browse files Browse the repository at this point in the history
  • Loading branch information
@d-ronnqvist
d-ronnqvist committed Dec 3, 2024
1 parent 6211801 commit 621a6ee
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 39 deletions.
23 changes: 21 additions & 2 deletions Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,16 @@

import Foundation

#if canImport(os)
import os
#endif

package enum ConvertActionConverter {
#if canImport(os)
static package let signposter = OSSignposter(subsystem: "org.swift.docc", category: "Convert")
#else
static package let signposter = NoOpSignposterShim()
#endif

/// Converts the documentation bundle in the given context and passes its output to a given consumer.
///
Expand All @@ -30,8 +39,12 @@ package enum ConvertActionConverter {
emitDigest: Bool,
documentationCoverageOptions: DocumentationCoverageOptions
) throws -> [Problem] {
let signposter = Self.signposter

defer {
context.diagnosticEngine.flush()
signposter.withIntervalSignpost("Display diagnostics", id: signposter.makeSignpostID()) {
context.diagnosticEngine.flush()
}
}

let processingDurationMetric = benchmark(begin: Benchmark.Duration(id: "documentation-processing"))
Expand All @@ -47,7 +60,9 @@ package enum ConvertActionConverter {
}

// Precompute the render context
let renderContext = RenderContext(documentationContext: context, bundle: bundle)
let renderContext = signposter.withIntervalSignpost("Build RenderContext", id: signposter.makeSignpostID()) {
RenderContext(documentationContext: context, bundle: bundle)
}
try outputConsumer.consume(renderReferenceStore: renderContext.store)

// Copy images, sample files, and other static assets.
Expand Down Expand Up @@ -89,6 +104,8 @@ package enum ConvertActionConverter {
let resultsSyncQueue = DispatchQueue(label: "Convert Serial Queue", qos: .unspecified, attributes: [])
let resultsGroup = DispatchGroup()

let renderSignpostHandle = signposter.beginInterval("Render", id: signposter.makeSignpostID(), "Render \(context.knownPages.count) pages")

var conversionProblems: [Problem] = context.knownPages.concurrentPerform { identifier, results in
// If cancelled skip all concurrent conversion work in this block.
guard !Task.isCancelled else { return }
Expand Down Expand Up @@ -146,6 +163,8 @@ package enum ConvertActionConverter {
// Wait for any concurrent updates to complete.
resultsGroup.wait()

signposter.endInterval("Render", renderSignpostHandle)

guard !Task.isCancelled else { return [] }

// Write various metadata
Expand Down
82 changes: 63 additions & 19 deletions Sources/SwiftDocC/Infrastructure/DocumentationContext.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public typealias BundleIdentifier = String
/// - ``parents(of:)``
///
public class DocumentationContext {
private let signposter = ConvertActionConverter.signposter

/// An error that's encountered while interacting with a ``SwiftDocC/DocumentationContext``.
public enum ContextError: DescribedError {
Expand Down Expand Up @@ -563,6 +564,11 @@ public class DocumentationContext {
Attempt to resolve links in curation-only documentation, converting any ``TopicReferences`` from `.unresolved` to `.resolved` where possible.
*/
private func resolveLinks(curatedReferences: Set<ResolvedTopicReference>, bundle: DocumentationBundle) {
let signpostHandle = signposter.beginInterval("Resolve links", id: signposter.makeSignpostID())
defer {
signposter.endInterval("Resolve links", signpostHandle)
}

let references = Array(curatedReferences)
let results = Synchronized<[LinkResolveResult]>([])
results.sync({ $0.reserveCapacity(references.count) })
Expand Down Expand Up @@ -708,6 +714,11 @@ public class DocumentationContext {
tutorialArticles: [SemanticResult<TutorialArticle>],
bundle: DocumentationBundle
) {
let signpostHandle = signposter.beginInterval("Resolve links", id: signposter.makeSignpostID())
defer {
signposter.endInterval("Resolve links", signpostHandle)
}

let sourceLanguages = soleRootModuleReference.map { self.sourceLanguages(for: $0) } ?? [.swift]

// Tutorial table-of-contents
Expand Down Expand Up @@ -1147,6 +1158,11 @@ public class DocumentationContext {
) throws {
// Making sure that we correctly let decoding memory get released, do not remove the autorelease pool.
try autoreleasepool {
let signpostHandle = signposter.beginInterval("Register symbols", id: signposter.makeSignpostID())
defer {
signposter.endInterval("Register symbols", signpostHandle)
}

/// We need only unique relationships so we'll collect them in a set.
var combinedRelationshipsBySelector = [UnifiedSymbolGraph.Selector: Set<SymbolGraph.Relationship>]()
/// Also track the unique relationships across all languages and platforms
Expand All @@ -1157,7 +1173,9 @@ public class DocumentationContext {
var moduleReferences = [String: ResolvedTopicReference]()

// Build references for all symbols in all of this module's symbol graphs.
let symbolReferences = linkResolver.localResolver.referencesForSymbols(in: symbolGraphLoader.unifiedGraphs, bundle: bundle, context: self)
let symbolReferences = signposter.withIntervalSignpost("Disambiguate references") {
linkResolver.localResolver.referencesForSymbols(in: symbolGraphLoader.unifiedGraphs, bundle: bundle, context: self)
}

// Set the index and cache storage capacity to avoid ad-hoc storage resizing.
documentationCache.reserveCapacity(symbolReferences.count)
Expand Down Expand Up @@ -1223,7 +1241,9 @@ public class DocumentationContext {
let moduleSymbolReference = SymbolReference(moduleName, interfaceLanguages: moduleInterfaceLanguages, defaultSymbol: moduleSymbol)
moduleReference = ResolvedTopicReference(symbolReference: moduleSymbolReference, moduleName: moduleName, bundle: bundle)

addSymbolsToTopicGraph(symbolGraph: unifiedSymbolGraph, url: fileURL, symbolReferences: symbolReferences, moduleReference: moduleReference)
signposter.withIntervalSignpost("Add symbols to topic graph", id: signposter.makeSignpostID()) {
addSymbolsToTopicGraph(symbolGraph: unifiedSymbolGraph, url: fileURL, symbolReferences: symbolReferences, moduleReference: moduleReference)
}

// For inherited symbols we remove the source docs (if inheriting docs is disabled) before creating their documentation nodes.
for (_, relationships) in unifiedSymbolGraph.relationshipsByLanguage {
Expand Down Expand Up @@ -1375,15 +1395,17 @@ public class DocumentationContext {
)

// Parse and prepare the nodes' content concurrently.
let updatedNodes = Array(documentationCache.symbolReferences).concurrentMap { finalReference in
// Match the symbol's documentation extension and initialize the node content.
let match = uncuratedDocumentationExtensions[finalReference]
let updatedNode = nodeWithInitializedContent(reference: finalReference, match: match)

return ((
node: updatedNode,
matchedArticleURL: match?.source
))
let updatedNodes = signposter.withIntervalSignpost("Parse symbol markup", id: signposter.makeSignpostID()) {
Array(documentationCache.symbolReferences).concurrentMap { finalReference in
// Match the symbol's documentation extension and initialize the node content.
let match = uncuratedDocumentationExtensions[finalReference]
let updatedNode = nodeWithInitializedContent(reference: finalReference, match: match)

return ((
node: updatedNode,
matchedArticleURL: match?.source
))
}
}

// Update cache with up-to-date nodes
Expand Down Expand Up @@ -2177,9 +2199,16 @@ public class DocumentationContext {
)

do {
try symbolGraphLoader.loadAll()
let pathHierarchy = PathHierarchy(symbolGraphLoader: symbolGraphLoader, bundleName: urlReadablePath(bundle.displayName), knownDisambiguatedPathComponents: configuration.convertServiceConfiguration.knownDisambiguatedSymbolPathComponents)
hierarchyBasedResolver = PathHierarchyBasedLinkResolver(pathHierarchy: pathHierarchy)
try signposter.withIntervalSignpost("Load symbols", id: signposter.makeSignpostID()) {
try symbolGraphLoader.loadAll()
}
hierarchyBasedResolver = signposter.withIntervalSignpost("Build PathHierarchy", id: signposter.makeSignpostID()) {
PathHierarchyBasedLinkResolver(pathHierarchy: PathHierarchy(
symbolGraphLoader: symbolGraphLoader,
bundleName: urlReadablePath(bundle.displayName),
knownDisambiguatedPathComponents: configuration.convertServiceConfiguration.knownDisambiguatedSymbolPathComponents
))
}
} catch {
// Pipe the error out of the dispatch queue.
discoveryError.sync({
Expand All @@ -2191,7 +2220,9 @@ public class DocumentationContext {
// First, all the resources are added since they don't reference anything else.
discoveryGroup.async(queue: discoveryQueue) { [unowned self] in
do {
try self.registerMiscResources(from: bundle)
try signposter.withIntervalSignpost("Load resources", id: signposter.makeSignpostID()) {
try self.registerMiscResources(from: bundle)
}
} catch {
// Pipe the error out of the dispatch queue.
discoveryError.sync({
Expand All @@ -2215,7 +2246,9 @@ public class DocumentationContext {

discoveryGroup.async(queue: discoveryQueue) { [unowned self] in
do {
result = try self.registerDocuments(from: bundle)
result = try signposter.withIntervalSignpost("Load documents", id: signposter.makeSignpostID()) {
try self.registerDocuments(from: bundle)
}
} catch {
// Pipe the error out of the dispatch queue.
discoveryError.sync({
Expand All @@ -2226,7 +2259,9 @@ public class DocumentationContext {

discoveryGroup.async(queue: discoveryQueue) { [unowned self] in
do {
try linkResolver.loadExternalResolvers(dependencyArchives: configuration.externalDocumentationConfiguration.dependencyArchives)
try signposter.withIntervalSignpost("Load external resolvers", id: signposter.makeSignpostID()) {
try linkResolver.loadExternalResolvers(dependencyArchives: configuration.externalDocumentationConfiguration.dependencyArchives)
}
} catch {
// Pipe the error out of the dispatch queue.
discoveryError.sync({
Expand Down Expand Up @@ -2361,7 +2396,9 @@ public class DocumentationContext {
try shouldContinueRegistration()

// Fourth, automatically curate all symbols that haven't been curated manually
let automaticallyCurated = autoCurateSymbolsInTopicGraph()
let automaticallyCurated = signposter.withIntervalSignpost("Auto-curate symbols ", id: signposter.makeSignpostID()) {
autoCurateSymbolsInTopicGraph()
}

// Crawl the rest of the symbols that haven't been crawled so far in hierarchy pre-order.
allCuratedReferences = try crawlSymbolCuration(in: automaticallyCurated.map(\.symbol), bundle: bundle, initial: allCuratedReferences)
Expand Down Expand Up @@ -2407,7 +2444,9 @@ public class DocumentationContext {
}

// Seventh, the complete topic graph—with all nodes and all edges added—is analyzed.
topicGraphGlobalAnalysis()
signposter.withIntervalSignpost("Analyze topic graph", id: signposter.makeSignpostID()) {
topicGraphGlobalAnalysis()
}

preResolveModuleNames()
}
Expand Down Expand Up @@ -2606,6 +2645,11 @@ public class DocumentationContext {
/// - Returns: The references of all the symbols that were curated.
@discardableResult
func crawlSymbolCuration(in references: [ResolvedTopicReference], bundle: DocumentationBundle, initial: Set<ResolvedTopicReference> = []) throws -> Set<ResolvedTopicReference> {
let signpostHandle = signposter.beginInterval("Curate symbols", id: signposter.makeSignpostID())
defer {
signposter.endInterval("Curate symbols", signpostHandle)
}

var crawler = DocumentationCurator(in: self, bundle: bundle, initial: initial)

for reference in references {
Expand Down
9 changes: 8 additions & 1 deletion Sources/SwiftDocC/Infrastructure/Symbol Graph/SymbolGraphLoader.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ struct SymbolGraphLoader {
///
/// - Throws: If loading and decoding any of the symbol graph files throws, this method re-throws one of the encountered errors.
mutating func loadAll() throws {
let signposter = ConvertActionConverter.signposter

let loadingLock = Lock()

var loadedGraphs = [URL: (usesExtensionSymbolFormat: Bool?, graph: SymbolKit.SymbolGraph)]()
Expand Down Expand Up @@ -118,6 +120,8 @@ struct SymbolGraphLoader {
}
#endif

let numberOfSymbolGraphs = bundle.symbolGraphURLs.count
let decodeSignpostHandle = signposter.beginInterval("Decode symbol graphs", id: signposter.makeSignpostID(), "Decode \(numberOfSymbolGraphs) symbol graphs")
switch decodingStrategy {
case .concurrentlyAllFiles:
// Concurrently load and decode all symbol graphs
Expand All @@ -127,12 +131,14 @@ struct SymbolGraphLoader {
// Serially load and decode all symbol graphs, each one in concurrent batches.
bundle.symbolGraphURLs.forEach(loadGraphAtURL)
}
signposter.endInterval("Decode symbol graphs", decodeSignpostHandle)

// define an appropriate merging strategy based on the graph formats
let foundGraphUsingExtensionSymbolFormat = loadedGraphs.values.map(\.usesExtensionSymbolFormat).contains(true)

let usingExtensionSymbolFormat = foundGraphUsingExtensionSymbolFormat


let mergeSignpostHandle = signposter.beginInterval("Build unified symbol graph", id: signposter.makeSignpostID())
let graphLoader = GraphCollector(extensionGraphAssociationStrategy: usingExtensionSymbolFormat ? .extendingGraph : .extendedGraph)

// feed the loaded graphs into the `graphLoader`
Expand All @@ -150,6 +156,7 @@ struct SymbolGraphLoader {
(self.unifiedGraphs, self.graphLocations) = graphLoader.finishLoading(
createOverloadGroups: FeatureFlags.current.isExperimentalOverloadedSymbolPresentationEnabled
)
signposter.endInterval("Build unified symbol graph", mergeSignpostHandle)

for var unifiedGraph in unifiedGraphs.values {
var defaultUnavailablePlatforms = [PlatformName]()
Expand Down
52 changes: 35 additions & 17 deletions Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import SwiftDocC

/// An action that converts a source bundle into compiled documentation.
public struct ConvertAction: AsyncAction {
private let signposter = ConvertActionConverter.signposter

let rootURL: URL?
let targetDirectory: URL
let htmlTemplateDirectory: URL?
Expand Down Expand Up @@ -165,12 +167,15 @@ public struct ConvertAction: AsyncAction {
}
configuration.externalDocumentationConfiguration.dependencyArchives = dependencies

let inputProvider = DocumentationContext.InputsProvider(fileManager: fileManager)
let (bundle, dataProvider) = try inputProvider.inputsAndDataProvider(
startingPoint: documentationBundleURL,
allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories,
options: bundleDiscoveryOptions
)
let (bundle, dataProvider) = try signposter.withIntervalSignpost("Discover inputs", id: signposter.makeSignpostID()) {
try DocumentationContext.InputsProvider(fileManager: fileManager)
.inputsAndDataProvider(
startingPoint: documentationBundleURL,
allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories,
options: bundleDiscoveryOptions
)
}

self.configuration = configuration

self.bundle = bundle
Expand Down Expand Up @@ -206,6 +211,11 @@ public struct ConvertAction: AsyncAction {
}

private func _perform(logHandle: inout LogHandle, temporaryFolder: URL) async throws -> (ActionResult, DocumentationContext) {
let convertSignpostHandle = signposter.beginInterval("Convert", id: signposter.makeSignpostID())
defer {
signposter.endInterval("Convert", convertSignpostHandle)
}

// Add the default diagnostic console writer now that we know what log handle it should write to.
if !diagnosticEngine.hasConsumer(matching: { $0 is DiagnosticConsoleWriter }) {
diagnosticEngine.add(
Expand Down Expand Up @@ -278,7 +288,9 @@ public struct ConvertAction: AsyncAction {

let indexer = try Indexer(outputURL: temporaryFolder, bundleID: bundle.id)

let context = try DocumentationContext(bundle: bundle, dataProvider: dataProvider, diagnosticEngine: diagnosticEngine, configuration: configuration)
let context = try signposter.withIntervalSignpost("Register", id: signposter.makeSignpostID()) {
try DocumentationContext(bundle: bundle, dataProvider: dataProvider, diagnosticEngine: diagnosticEngine, configuration: configuration)
}

let outputConsumer = ConvertFileWritingConsumer(
targetFolder: temporaryFolder,
Expand All @@ -304,14 +316,16 @@ public struct ConvertAction: AsyncAction {
let analysisProblems: [Problem]
let conversionProblems: [Problem]
do {
conversionProblems = try ConvertActionConverter.convert(
bundle: bundle,
context: context,
outputConsumer: outputConsumer,
sourceRepository: sourceRepository,
emitDigest: emitDigest,
documentationCoverageOptions: documentationCoverageOptions
)
conversionProblems = try signposter.withIntervalSignpost("Process") {
try ConvertActionConverter.convert(
bundle: bundle,
context: context,
outputConsumer: outputConsumer,
sourceRepository: sourceRepository,
emitDigest: emitDigest,
documentationCoverageOptions: documentationCoverageOptions
)
}
analysisProblems = context.problems
} catch {
if emitDigest {
Expand Down Expand Up @@ -366,7 +380,9 @@ public struct ConvertAction: AsyncAction {

// Always emit a JSON representation of the index but only emit the LMDB
// index if the user has explicitly opted in with the `--emit-lmdb-index` flag.
let indexerProblems = indexer.finalize(emitJSON: true, emitLMDB: buildLMDBIndex)
let indexerProblems = signposter.withIntervalSignpost("Finalize navigator index") {
indexer.finalize(emitJSON: true, emitLMDB: buildLMDBIndex)
}
postConversionProblems.append(contentsOf: indexerProblems)

benchmark(end: finalizeNavigationIndexMetric)
Expand Down Expand Up @@ -436,6 +452,8 @@ public struct ConvertAction: AsyncAction {
}

func moveOutput(from: URL, to: URL) throws {
return try Self.moveOutput(from: from, to: to, fileManager: fileManager)
try signposter.withIntervalSignpost("Move output") {
try Self.moveOutput(from: from, to: to, fileManager: fileManager)
}
}
}

0 comments on commit 621a6ee

Please sign in to comment.