New AST types (#8263)

Introduce new AST type; use it to replace current uses of AstExtended; for now edits and synchronization are implemented on the old mechanisms (text edits / RelativeRange id map).

# Important Notes
- Edit-related code is commented out until the next PR because it is incompatible with the transitional IdMap-based synchronization.

app/gui2/src/components/CodeEditor.vue

@@ -58,7 +58,8 @@ const expressionUpdatesDiagnostics = computed(() => {
     if (!update) continue
     const node = nodeMap.get(id)
     if (!node) continue
-    const [from, to] = node.rootSpan.span()
+    if (!node.rootSpan.astExtended) continue
+    const [from, to] = node.rootSpan.astExtended.span()
     switch (update.payload.type) {
       case 'Panic': {
         diagnostics.push({ from, to, message: update.payload.message, severity: 'error' })
@@ -102,7 +103,10 @@ watchEffect(() => {
           const astSpan = ast.span()
           let foundNode: ExprId | undefined
           for (const [id, node] of graphStore.db.nodeIdToNode.entries()) {
-            if (rangeEncloses(node.rootSpan.span(), astSpan)) {
+            if (
+              node.rootSpan.astExtended &&
+              rangeEncloses(node.rootSpan.astExtended.span(), astSpan)
+            ) {
               foundNode = id
               break
             }

app/gui2/src/components/ComponentBrowser/input.ts

@@ -9,7 +9,7 @@ import {
   type SuggestionEntry,
   type Typename,
 } from '@/stores/suggestionDatabase/entry'
-import { Ast, AstExtended, astContainingChar } from '@/util/ast'
+import { RawAst, RawAstExtended, astContainingChar } from '@/util/ast'
 import { AliasAnalyzer } from '@/util/ast/aliasAnalysis'
 import { GeneralOprApp, type OperatorChain } from '@/util/ast/opr'
 import { MappedSet } from '@/util/containers'
@@ -42,11 +42,14 @@ export type EditingContext =
   // Suggestion should replace given identifier.
   | {
       type: 'changeIdentifier'
-      identifier: AstExtended<Ast.Tree.Ident, false>
+      identifier: RawAstExtended<RawAst.Tree.Ident, false>
       oprApp?: GeneralOprApp<false>
     }
   // Suggestion should replace given literal.
-  | { type: 'changeLiteral'; literal: AstExtended<Ast.Tree.TextLiteral | Ast.Tree.Number, false> }
+  | {
+      type: 'changeLiteral'
+      literal: RawAstExtended<RawAst.Tree.TextLiteral | RawAst.Tree.Number, false>
+    }
 
 /** An atomic change to the user input. */
 interface Change {
@@ -62,7 +65,7 @@ export function useComponentBrowserInput(
 ) {
   const code = ref('')
   const selection = ref({ start: 0, end: 0 })
-  const ast = computed(() => AstExtended.parse(code.value))
+  const ast = computed(() => RawAstExtended.parse(code.value))
 
   const context: ComputedRef<EditingContext> = computed(() => {
     const cursorPosition = selection.value.start
@@ -75,17 +78,20 @@ export function useComponentBrowserInput(
     const leaf = editedAst.next()
     if (leaf.done) return { type: 'insert', position: cursorPosition }
     switch (leaf.value.inner.type) {
-      case Ast.Tree.Type.Ident:
+      case RawAst.Tree.Type.Ident:
         return {
           type: 'changeIdentifier',
-          identifier: leaf.value as AstExtended<Ast.Tree.Ident, false>,
+          identifier: leaf.value as RawAstExtended<RawAst.Tree.Ident, false>,
           ...readOprApp(editedAst.next(), leaf.value),
         }
-      case Ast.Tree.Type.TextLiteral:
-      case Ast.Tree.Type.Number:
+      case RawAst.Tree.Type.TextLiteral:
+      case RawAst.Tree.Type.Number:
         return {
           type: 'changeLiteral',
-          literal: leaf.value as AstExtended<Ast.Tree.TextLiteral | Ast.Tree.Number, false>,
+          literal: leaf.value as RawAstExtended<
+            RawAst.Tree.TextLiteral | RawAst.Tree.Number,
+            false
+          >,
         }
       default:
         return {
@@ -140,15 +146,15 @@ export function useComponentBrowserInput(
   const imports = ref<{ context: string; info: RequiredImport }[]>([])
 
   function readOprApp(
-    leafParent: IteratorResult<AstExtended<Ast.Tree, false>>,
-    editedAst?: AstExtended<Ast.Tree, false>,
+    leafParent: IteratorResult<RawAstExtended<RawAst.Tree, false>>,
+    editedAst?: RawAstExtended<RawAst.Tree, false>,
   ): {
     oprApp?: GeneralOprApp<false>
   } {
     if (leafParent.done) return {}
     switch (leafParent.value.inner.type) {
-      case Ast.Tree.Type.OprApp:
-      case Ast.Tree.Type.OperatorBlockApplication: {
+      case RawAst.Tree.Type.OprApp:
+      case RawAst.Tree.Type.OperatorBlockApplication: {
         const generalized = new GeneralOprApp(leafParent.value as OperatorChain<false>)
         const opr = generalized.lastOpr()
         if (opr == null) return {}
@@ -169,7 +175,7 @@ export function useComponentBrowserInput(
     accessOpr: GeneralOprApp<false>,
   ): { type: 'known'; typename: Typename } | { type: 'unknown' } | null {
     if (accessOpr.lhs == null) return null
-    if (!accessOpr.lhs.isTree(Ast.Tree.Type.Ident)) return null
+    if (!accessOpr.lhs.isTree(RawAst.Tree.Type.Ident)) return null
     if (accessOpr.apps.length > 1) return null
     if (internalUsages.value.has(accessOpr.lhs.span())) return { type: 'unknown' }
     const ident = accessOpr.lhs.repr()
@@ -194,12 +200,12 @@ export function useComponentBrowserInput(
    * @param code The code from which `opr` was generated.
    * @returns If all path segments are identifiers, return them
    */
-  function qnIdentifiers(opr: GeneralOprApp<false>): AstExtended<Ast.Tree.Ident, false>[] {
+  function qnIdentifiers(opr: GeneralOprApp<false>): RawAstExtended<RawAst.Tree.Ident, false>[] {
     const operandsAsIdents = Array.from(opr.operandsOfLeftAssocOprChain('.'), (operand) =>
-      operand?.type === 'ast' && operand.ast.isTree(Ast.Tree.Type.Ident) ? operand.ast : null,
+      operand?.type === 'ast' && operand.ast.isTree(RawAst.Tree.Type.Ident) ? operand.ast : null,
     ).slice(0, -1)
     if (operandsAsIdents.some((optIdent) => optIdent == null)) return []
-    else return operandsAsIdents as AstExtended<Ast.Tree.Ident, false>[]
+    else return operandsAsIdents as RawAstExtended<RawAst.Tree.Ident, false>[]
   }
 
   /** Apply given suggested entry to the input. */

app/gui2/src/components/GraphEditor/NodeWidget.vue

@@ -10,7 +10,7 @@ import {
   provideWidgetUsageInfo,
   usageKeyForInput,
 } from '@/providers/widgetUsageInfo'
-import { AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 import { computed, proxyRefs, ref } from 'vue'
 
 const props = defineProps<{ input: WidgetInput; nest?: boolean }>()
@@ -25,8 +25,8 @@ const usageKey = computed(() => usageKeyForInput(props.input))
 const sameInputAsParent = computed(() => parentUsageInfo?.usageKey === usageKey.value)
 
 const whitespace = computed(() =>
-  !sameInputAsParent.value && props.input instanceof AstExtended
-    ? ' '.repeat(props.input.whitespaceLength() ?? 0)
+  !sameInputAsParent.value && props.input instanceof Ast.Ast
+    ? ' '.repeat(props.input.astExtended?.whitespaceLength() ?? 0)
     : '',
 )
 
@@ -65,8 +65,9 @@ provideWidgetUsageInfo(
   }),
 )
 const spanStart = computed(() => {
-  if (!(props.input instanceof AstExtended)) return undefined
-  return props.input.span()[0] - tree.nodeSpanStart - whitespace.value.length
+  if (!(props.input instanceof Ast.Ast)) return undefined
+  if (props.input.astExtended == null) return undefined
+  return props.input.astExtended.span()[0] - tree.nodeSpanStart - whitespace.value.length
 })
 </script>
 

app/gui2/src/components/GraphEditor/NodeWidgetTree.vue

@@ -1,17 +1,16 @@
 <script setup lang="ts">
-import { Tree } from '@/generated/ast'
 import { ForcePort } from '@/providers/portInfo'
 import { provideWidgetTree } from '@/providers/widgetTree'
 import { useGraphStore } from '@/stores/graph'
 import { useTransitioning } from '@/util/animation'
-import type { AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 import { computed, toRef } from 'vue'
 import NodeWidget from './NodeWidget.vue'
 
-const props = defineProps<{ ast: AstExtended }>()
+const props = defineProps<{ ast: Ast.Ast }>()
 const graph = useGraphStore()
 const rootPort = computed(() => {
-  return props.ast.isTree(Tree.Type.Ident) && !graph.db.isKnownFunctionCall(props.ast.astId)
+  return props.ast instanceof Ast.Ident && !graph.db.isKnownFunctionCall(props.ast.astId)
     ? new ForcePort(props.ast)
     : props.ast
 })
@@ -34,9 +33,9 @@ provideWidgetTree(toRef(props, 'ast'), layoutTransitions.active)
 </script>
 
 <template>
-  <span class="NodeWidgetTree" spellcheck="false" v-on="layoutTransitions.events">
+  <div class="NodeWidgetTree" spellcheck="false" v-on="layoutTransitions.events">
     <NodeWidget :input="rootPort" />
-  </span>
+  </div>
 </template>
 
 <style scoped>
@@ -46,7 +45,7 @@ provideWidgetTree(toRef(props, 'ast'), layoutTransitions.active)
 
   outline: none;
   height: 24px;
-  display: inline-flex;
+  display: flex;
   align-items: center;
 
   & :deep(span) {

app/gui2/src/components/GraphEditor/widgets/WidgetApplication.vue

@@ -2,15 +2,13 @@
 import NodeWidget from '@/components/GraphEditor/NodeWidget.vue'
 import { ForcePort } from '@/providers/portInfo'
 import { defineWidget, widgetProps } from '@/providers/widgetRegistry'
-import { AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 import { ArgumentApplication } from '@/util/callTree'
 import { computed } from 'vue'
 
 const props = defineProps(widgetProps(widgetDefinition))
 const targetMaybePort = computed(() =>
-  props.input.target instanceof AstExtended
-    ? new ForcePort(props.input.target)
-    : props.input.target,
+  props.input.target instanceof Ast.Ast ? new ForcePort(props.input.target) : props.input.target,
 )
 </script>
 

app/gui2/src/components/GraphEditor/widgets/WidgetBlank.vue

@@ -1,11 +1,11 @@
 <script setup lang="ts">
 import { Score, defineWidget, widgetProps } from '@/providers/widgetRegistry'
-import { Ast, AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 const _props = defineProps(widgetProps(widgetDefinition))
 </script>
 
 <script lang="ts">
-export const widgetDefinition = defineWidget(AstExtended.isToken(Ast.Token.Type.Wildcard), {
+export const widgetDefinition = defineWidget((ast) => ast instanceof Ast.Wildcard, {
   priority: 10,
   score: Score.Good,
 })

app/gui2/src/components/GraphEditor/widgets/WidgetCheckbox.vue

@@ -1,17 +1,16 @@
 <script setup lang="ts">
 import CheckboxWidget from '@/components/widgets/CheckboxWidget.vue'
-import { Tree } from '@/generated/ast'
 import { Score, defineWidget, widgetProps } from '@/providers/widgetRegistry'
 import { useGraphStore } from '@/stores/graph'
-import { AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 import { computed } from 'vue'
 
 const props = defineProps(widgetProps(widgetDefinition))
 
 const graph = useGraphStore()
 const value = computed({
   get() {
-    return props.input.repr().endsWith('True') ?? false
+    return props.input.code().endsWith('True') ?? false
   },
   set(value) {
     const node = getRawBoolNode(props.input)
@@ -23,23 +22,17 @@ const value = computed({
 </script>
 
 <script lang="ts">
-function getRawBoolNode(ast: AstExtended) {
+function getRawBoolNode(ast: Ast.Ast) {
   const candidate =
-    ast.isTree(Tree.Type.OprApp) && ast.repr().startsWith('Boolean.')
-      ? ast.tryMap((t) => t.rhs)
-      : ast
-  if (
-    candidate &&
-    candidate.isTree(Tree.Type.Ident) &&
-    ['True', 'False'].includes(candidate.repr())
-  ) {
+    ast instanceof Ast.PropertyAccess && ast.lhs?.code() === 'Boolean' ? ast.rhs : ast
+  if (candidate instanceof Ast.Ident && ['True', 'False'].includes(candidate.code())) {
     return candidate
   }
   return null
 }
 
 export const widgetDefinition = defineWidget(
-  AstExtended.isTree([Tree.Type.OprApp, Tree.Type.Ident]),
+  (input) => input instanceof Ast.PropertyAccess || input instanceof Ast.Ident,
   {
     priority: 10,
     score: (props) => {

app/gui2/src/components/GraphEditor/widgets/WidgetFunction.vue

@@ -1,10 +1,9 @@
 <script setup lang="ts">
 import NodeWidget from '@/components/GraphEditor/NodeWidget.vue'
-import { Tree } from '@/generated/ast'
 import { injectFunctionInfo, provideFunctionInfo } from '@/providers/functionInfo'
 import { Score, defineWidget, widgetProps } from '@/providers/widgetRegistry'
 import { useGraphStore } from '@/stores/graph'
-import { AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 import { ArgumentApplication } from '@/util/callTree'
 import { computed, proxyRefs } from 'vue'
 
@@ -19,10 +18,10 @@ provideFunctionInfo(
 )
 
 const application = computed(() => {
-  const astId = props.input.astId
-  if (astId == null) return props.input
+  const input: Ast.Ast = props.input
+  const astId = input.astId
   const info = graph.db.getMethodCallInfo(astId)
-  const interpreted = ArgumentApplication.Interpret(props.input, info == null)
+  const interpreted = ArgumentApplication.Interpret(input, info == null)
 
   const noArgsCall =
     interpreted.kind === 'prefix' ? graph.db.getMethodCall(interpreted.func.astId) : undefined
@@ -38,7 +37,7 @@ const application = computed(() => {
 </script>
 <script lang="ts">
 export const widgetDefinition = defineWidget(
-  AstExtended.isTree([Tree.Type.App, Tree.Type.NamedApp, Tree.Type.Ident, Tree.Type.OprApp]),
+  (ast) => ast instanceof Ast.App || ast instanceof Ast.Ident || ast instanceof Ast.OprApp,
   {
     priority: -10,
     score: (props, db) => {
@@ -52,13 +51,13 @@ export const widgetDefinition = defineWidget(
       // and to resolve the infix call as its own application.
       if (prevFunctionState?.callId === ast.astId) return Score.Mismatch
 
-      if (ast.isTree([Tree.Type.App, Tree.Type.NamedApp, Tree.Type.OprApp])) return Score.Perfect
+      if (ast instanceof Ast.App || ast instanceof Ast.OprApp) return Score.Perfect
 
       const info = db.getMethodCallInfo(ast.astId)
       if (
         prevFunctionState != null &&
         info?.staticallyApplied === true &&
-        props.input.isTree(Tree.Type.Ident)
+        ast instanceof Ast.Ident
       ) {
         return Score.Mismatch
       }

app/gui2/src/components/GraphEditor/widgets/WidgetHierarchy.vue

@@ -1,17 +1,17 @@
 <script setup lang="ts">
 import NodeWidget from '@/components/GraphEditor/NodeWidget.vue'
 import { defineWidget, widgetProps } from '@/providers/widgetRegistry'
-import { AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 import { computed } from 'vue'
 
 const props = defineProps(widgetProps(widgetDefinition))
 
-const spanClass = computed(() => props.input.treeTypeName())
+const spanClass = computed(() => props.input.typeName())
 const children = computed(() => [...props.input.children()])
 </script>
 
 <script lang="ts">
-export const widgetDefinition = defineWidget(AstExtended.isTree(), {
+export const widgetDefinition = defineWidget((expression) => expression instanceof Ast.Ast, {
   priority: 1001,
 })
 </script>

app/gui2/src/components/GraphEditor/widgets/WidgetNumber.vue

@@ -1,16 +1,15 @@
 <script setup lang="ts">
 import SliderWidget from '@/components/widgets/SliderWidget.vue'
-import { Tree } from '@/generated/ast'
 import { Score, defineWidget, widgetProps } from '@/providers/widgetRegistry'
 import { useGraphStore } from '@/stores/graph'
-import { AstExtended } from '@/util/ast'
+import { Ast } from '@/util/ast'
 import { computed } from 'vue'
 
 const props = defineProps(widgetProps(widgetDefinition))
 const graph = useGraphStore()
 const value = computed({
   get() {
-    return parseFloat(props.input.repr() ?? '')
+    return parseFloat(props.input.code() ?? '')
   },
   set(value) {
     const id = props.input.astId
@@ -21,22 +20,12 @@ const value = computed({
 
 <script lang="ts">
 export const widgetDefinition = defineWidget(
-  AstExtended.isTree([Tree.Type.UnaryOprApp, Tree.Type.Number]),
+  (input) =>
+    input instanceof Ast.NumericLiteral ||
+    (input instanceof Ast.NegationOprApp && input.argument instanceof Ast.NumericLiteral),
   {
     priority: 10,
-    score: (props) => {
-      if (props.input.isTree(Tree.Type.UnaryOprApp)) {
-        if (
-          props.input.map((t) => t.opr).repr() === '-' &&
-          props.input.tryMap((t) => t.rhs)?.isTree(Tree.Type.Number)
-        ) {
-          return Score.Perfect
-        }
-      } else if (props.input.isTree(Tree.Type.Number)) {
-        return Score.Perfect
-      }
-      return Score.Mismatch
-    },
+    score: Score.Perfect,
   },
 )
 </script>