GPU Aggregation (1/8): Aggregator and AggregationLayer (#8886)

modules/aggregation-layers/src/aggregation-layer-v9/aggregation-layer.ts

@@ -0,0 +1,91 @@
+import {
+  CompositeLayer,
+  LayerDataSource,
+  LayerContext,
+  UpdateParameters,
+  CompositeLayerProps,
+  Attribute,
+  AttributeManager
+} from '@deck.gl/core';
+import {Aggregator} from './aggregator';
+
+export type AggregationLayerProps<DataT> = CompositeLayerProps & {
+  data: LayerDataSource<DataT>;
+};
+
+export default abstract class AggregationLayer<
+  DataT,
+  ExtraPropsT extends {} = {}
+> extends CompositeLayer<Required<AggregationLayer<DataT>> & ExtraPropsT> {
+  static layerName = 'AggregationLayer';
+
+  state!: {
+    aggregator: Aggregator;
+  };
+
+  /** Allow this layer to participates in the draw cycle */
+  get isDrawable() {
+    return true;
+  }
+
+  /** Called to create an Aggregator instance */
+  abstract createAggregator(): Aggregator;
+  /** Called when some attributes change, a chance to mark Aggregator as dirty */
+  abstract onAttributeChange(id: string): void;
+
+  initializeState(): void {
+    this.getAttributeManager()!.remove(['instancePickingColors']);
+  }
+
+  // Override Layer.updateState to update the GPUAggregator instance
+  updateState(params: UpdateParameters<this>) {
+    super.updateState(params);
+
+    if (params.changeFlags.extensionsChanged) {
+      this.state.aggregator?.destroy();
+      this.state.aggregator = this.createAggregator();
+      this.getAttributeManager()!.invalidateAll();
+    }
+  }
+
+  // Override Layer.finalizeState to dispose the GPUAggregator instance
+  finalizeState(context: LayerContext) {
+    super.finalizeState(context);
+    this.state.aggregator.destroy();
+  }
+
+  // Override Layer.updateAttributes to update the aggregator
+  protected updateAttributes(changedAttributes: {[id: string]: Attribute}) {
+    const {aggregator} = this.state;
+    aggregator.setProps({
+      attributes: changedAttributes
+    });
+
+    for (const id in changedAttributes) {
+      this.onAttributeChange(id);
+    }
+
+    // In aggregator.update() the aggregator allocates the buffers to store its output
+    // These buffers will be exposed by aggregator.getResults() and passed to the sublayers
+    // Therefore update() must be called before renderLayers()
+    // CPUAggregator's output is populated right here in update()
+    // GPUAggregator's output is pre-allocated and populated in preDraw(), see comments below
+    aggregator.update();
+  }
+
+  draw({moduleParameters}) {
+    // GPU aggregation needs `moduleSettings` for projection/filter uniforms which are only accessible at draw time
+    // GPUAggregator's Buffers are pre-allocated during `update()` and passed down to the sublayer attributes in renderLayers()
+    // Although the Buffers have been bound to the sublayer's Model, their content are not populated yet
+    // GPUAggregator.preDraw() is called in the draw cycle here right before Buffers are used by sublayer.draw()
+    this.state.aggregator.preDraw({moduleSettings: moduleParameters});
+  }
+
+  // override CompositeLayer._getAttributeManager to create AttributeManager instance
+  _getAttributeManager() {
+    return new AttributeManager(this.context.device, {
+      id: this.props.id,
+      stats: this.context.stats
+    });
+  }
+}

modules/aggregation-layers/src/aggregation-layer-v9/aggregator.ts

@@ -0,0 +1,79 @@
+import type {Attribute, BinaryAttribute} from '@deck.gl/core';
+
+/** Method used to reduce a list of values to one number */
+export type AggregationOperation = 'SUM' | 'MEAN' | 'MIN' | 'MAX' | 'COUNT';
+
+/** Baseline inputs to an Aggregator */
+export type AggregationProps = {
+  /** Number of data points */
+  pointCount: number;
+  /** The input data */
+  attributes: {[id: string]: Attribute};
+  /** How to aggregate the values inside a bin, defined for each channel */
+  operations: AggregationOperation[];
+  /** Additional options to control bin sorting, e.g. bin size */
+  binOptions: Record<string, number | number[]>;
+};
+
+/** Descriptor of an aggregated bin */
+export type AggregatedBin = {
+  /** The unique identifier of the bin */
+  id: number[];
+  /** Aggregated values by channel */
+  value: number[];
+  /** Count of data points in this bin */
+  count: number;
+};
+
+/**
+ * The Aggregator interface describes a class that performs aggregation.
+ *
+ * _Aggregation_ is a 2-step process:
+ * 1. Sort: Group a collection of _data points_ by some property into _bins_.
+ * 2. Aggregate: for each _bin_, calculate a numeric output (_result_) from some metrics (_values_) from all its members.
+ *    Multiple results can be obtained independently (_channels_).
+ *
+ * An implementation of the _Aggregator_ interface takes the following inputs:
+ * - The number of data points
+ * - The group that each data point belongs to, by mapping each data point to a _binId_ (integer or array of integers)
+ * - The values to aggregate, by mapping each data point in each channel to one _value_ (number)
+ * - The method (_aggregationOperation_) to reduce a list of values to one number, such as SUM
+ *
+ * And yields the following outputs:
+ * - The aggregated values (_result_) as a list of numbers for each channel, comprised of one number per bin
+ * - The [min, max] among all aggregated values (_domain_) for each channel
+ *
+ */
+export interface Aggregator {
+  /** Update aggregation props */
+  setProps(props: Partial<AggregationProps>): void;
+
+  /** Flags a channel to need update
+   * @param {number} channel - mark the given channel as dirty. If not provided, all channels will be updated.
+   */
+  setNeedsUpdate(channel?: number): void;
+
+  /** Called after props are set and before results are accessed */
+  update(): void;
+
+  /** Called before layer is drawn to screen. */
+  preDraw(params?: {moduleSettings: any}): void;
+
+  /** Dispose all allocated resources */
+  destroy(): void;
+
+  /** Get the number of bins */
+  get numBins(): number;
+
+  /** Returns an accessor to the bins. */
+  getBins(): BinaryAttribute | null;
+
+  /** Returns an accessor to the output for a given channel. */
+  getResult(channel: number): BinaryAttribute | null;
+
+  /** Returns the [min, max] of aggregated values for a given channel. */
+  getResultDomain(channel: number): [min: number, max: number];
+
+  /** Returns the information for a given bin. */
+  getBin(index: number): AggregatedBin | null;
+}

modules/core/src/lib/composite-layer.ts

@@ -41,6 +41,11 @@ export default abstract class CompositeLayer<PropsT extends {} = {}> extends Lay
     return true;
   }
 
+  /** `true` if the layer renders to screen */
+  get isDrawable(): boolean {
+    return false;
+  }
+
   /** Returns true if all async resources are loaded */
   get isLoaded(): boolean {
     return super.isLoaded && this.getSubLayers().every(layer => layer.isLoaded);

modules/core/src/lib/layer.ts

@@ -285,6 +285,11 @@ export default abstract class Layer<PropsT extends {} = {}> extends Component<
     return false;
   }
 
+  /** `true` if the layer renders to screen */
+  get isDrawable(): boolean {
+    return true;
+  }
+
   /** Updates selected state members and marks the layer for redraw */
   setState(partialState: any): void {
     this.setChangeFlags({stateChanged: true});
@@ -1009,6 +1014,10 @@ export default abstract class Layer<PropsT extends {} = {}> extends Component<
         extension.updateState.call(this, updateParams, extension);
       }
 
+      this.setNeedsRedraw();
+      // Check if attributes need recalculation
+      this._updateAttributes();
+
       const modelChanged = this.getModels()[0] !== oldModels[0];
       this._postUpdate(updateParams, modelChanged);
       // End subclass lifecycle methods
@@ -1252,10 +1261,6 @@ export default abstract class Layer<PropsT extends {} = {}> extends Component<
   protected _postUpdate(updateParams: UpdateParameters<Layer<PropsT>>, forceUpdate: boolean) {
     const {props, oldProps} = updateParams;
 
-    this.setNeedsRedraw();
-    // Check if attributes need recalculation
-    this._updateAttributes();
-
     // Note: Automatic instance count update only works for single layers
     const model = this.state.model as Model | undefined;
     if (model?.isInstanced) {

modules/core/src/passes/layers-pass.ts

@@ -251,7 +251,8 @@ export default class LayersPass extends Pass {
       }
       if (layer.isComposite) {
         renderStatus.compositeCount++;
-      } else if (shouldDrawLayer) {
+      }
+      if (layer.isDrawable && shouldDrawLayer) {
         // Draw the layer
         renderStatus.visibleCount++;