Drop xmlparsedata and replace it with getParseData

.github/actions/setup/action.yaml

@@ -29,8 +29,3 @@ runs:
       uses: r-lib/actions/setup-r@v2
       with:
         r-version: ${{ inputs.r-version  }}
-
-    - name: Install R packages
-      if: ${{ inputs.r-version != '' }}
-      shell: Rscript {0}
-      run: install.packages("xmlparsedata", repos="https://cloud.r-project.org/")

package.json

@@ -352,7 +352,6 @@
     "@types/n3": "^1.16.4",
     "@types/object-hash": "^3.0.6",
     "@types/tmp": "^0.2.6",
-    "@types/xml2js": "^0.4.14",
     "@typescript-eslint/eslint-plugin": "^6.13.2",
     "chai": "^4.3.10",
     "chai-as-promised": "^7.1.1",
@@ -394,7 +393,6 @@
     "ts-essentials": "^9.4.1",
     "tslog": "^4.9.2",
     "ws": "^8.16.0",
-    "xml2js": "^0.6.2",
     "xpath-ts2": "^1.4.2"
   }
 }

scripts/Dockerfile

@@ -18,8 +18,6 @@ LABEL author="Florian Sihler" git="https://github.com/Code-Inspect/flowr"
 WORKDIR /app
 
 RUN apk add --no-cache R
-# install the xmlparsedata package because we need it
-RUN Rscript -e "install.packages('xmlparsedata',repos='https://cloud.r-project.org/')"
 
 # we keep the package.json for module resolution
 COPY ./package.json ./scripts/demo.R LICENSE /app/

src/benchmark/slicer.ts

@@ -5,7 +5,7 @@
 
 import type {
 	NormalizedAst,
-	RParseRequestFromFile, RParseRequestFromText, TokenMap
+	RParseRequestFromFile, RParseRequestFromText
 } from '../r-bridge'
 import {
 	collectAllIds,
@@ -47,8 +47,6 @@ export const benchmarkLogger = log.getSubLogger({ name: 'benchmark' })
 export interface BenchmarkSlicerStats extends MergeableRecord {
 	/** the measurements obtained during the benchmark */
 	stats:     SlicerStats
-	/** the used token map when translating what was parsed from R */
-	tokenMap:  Record<string, string>
 	/** the initial and unmodified AST produced by the R side/the 'parse' step */
 	parse:     string
 	/** the normalized AST produced by the 'normalization' step, including its parent decoration */
@@ -86,7 +84,6 @@ export class BenchmarkSlicer {
 	private readonly shell: RShell
 	private stats:          SlicerStats | undefined
 	private loadedXml:      string | undefined
-	private tokenMap:       Record<string, string> | undefined
 	private dataflow:       DataflowInformation | undefined
 	private ai:             DataflowInformation | undefined
 	private normalizedAst:  NormalizedAst | undefined
@@ -101,10 +98,6 @@ export class BenchmarkSlicer {
 			'initialize R session',
 			() => new RShell()
 		)
-		this.commonMeasurements.measure(
-			'inject home path',
-			() => this.shell.tryToInjectHomeLibPath()
-		)
 	}
 
 	/**
@@ -114,27 +107,11 @@ export class BenchmarkSlicer {
 	public async init(request: RParseRequestFromFile | RParseRequestFromText) {
 		guard(this.stats === undefined, 'cannot initialize the slicer twice')
 
-
-		await this.commonMeasurements.measureAsync(
-			'ensure installation of xmlparsedata',
-			() => this.shell.ensurePackageInstalled('xmlparsedata', true),
-		)
-
-		this.tokenMap = await this.commonMeasurements.measureAsync(
-			'retrieve token map',
-			// with this being the first time, there is no preexisting caching!
-			() => this.shell.tokenMap()
-		)
-
 		this.stepper = new SteppingSlicer({
-			shell:   this.shell,
-			request: {
-				...request,
-				ensurePackageInstalled: true
-			},
+			shell:          this.shell,
+			request:        {...request},
 			stepOfInterest: LAST_STEP,
-			criterion:      [],
-			tokenMap:       this.tokenMap
+			criterion:      []
 		})
 
 		this.loadedXml = await this.measureCommonStep('parse', 'retrieve AST from R code')
@@ -316,8 +293,7 @@ export class BenchmarkSlicer {
 			stats:     this.stats,
 			parse:     this.loadedXml as string,
 			dataflow:  this.dataflow as DataflowInformation,
-			normalize: this.normalizedAst as NormalizedAst,
-			tokenMap:  this.tokenMap as TokenMap,
+			normalize: this.normalizedAst as NormalizedAst
 		}
 	}
 

src/benchmark/stats/print.ts

@@ -91,7 +91,6 @@ export function stats2string(stats: SummarizedSlicerStats): string {
 	let result = `
 Request: ${JSON.stringify(stats.request)}
 Shell init time:              ${print(stats.commonMeasurements,'initialize R session')}
-Retrieval of token map:       ${print(stats.commonMeasurements,'retrieve token map')}
 AST retrieval:                ${print(stats.commonMeasurements,'retrieve AST from R code')}
 AST normalization:            ${print(stats.commonMeasurements,'normalize R AST')}
 Dataflow creation:            ${print(stats.commonMeasurements,'produce dataflow information')}
@@ -137,7 +136,6 @@ export function ultimateStats2String(stats: UltimateSlicerStats): string {
 	return `
 Summarized: ${stats.totalRequests} requests and ${stats.totalSlices} slices
 Shell init time:              ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('initialize R session'))}
-Retrieval of token map:       ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('retrieve token map'))}
 AST retrieval:                ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('retrieve AST from R code'))}
 AST normalization:            ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('normalize R AST'))}
 Dataflow creation:            ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('produce dataflow information'))}

src/benchmark/stats/stats.ts

@@ -2,7 +2,7 @@ import type { SingleSlicingCriterion, SlicingCriteria } from '../../slicing'
 import type { NodeId, RParseRequestFromFile, RParseRequestFromText } from '../../r-bridge'
 import type { ReconstructionResult } from '../../slicing'
 
-export const CommonSlicerMeasurements = ['initialize R session', 'inject home path', 'ensure installation of xmlparsedata', 'retrieve token map', 'retrieve AST from R code', 'normalize R AST', 'produce dataflow information', 'run abstract interpretation', 'close R session', 'total'] as const
+export const CommonSlicerMeasurements = ['initialize R session', 'retrieve AST from R code', 'normalize R AST', 'produce dataflow information', 'run abstract interpretation', 'close R session', 'total'] as const
 export type CommonSlicerMeasurements = typeof CommonSlicerMeasurements[number]
 
 export const PerSliceMeasurements = ['static slicing', 'reconstruct code', 'total'] as const

src/cli/export-quads-app.ts

@@ -23,13 +23,9 @@ const options = processCommandLineArgs<QuadsCliOptions>('export-quads', [],{
 })
 
 const shell = new RShell()
-shell.tryToInjectHomeLibPath()
 
 async function writeQuadForSingleFile(request: RParseRequestFromFile, output: string) {
-	const normalized = await retrieveNormalizedAstFromRCode({
-		...request,
-		ensurePackageInstalled: true
-	}, shell)
+	const normalized = await retrieveNormalizedAstFromRCode(request, shell)
 	const serialized = serialize2quads(normalized.ast, { context: request.content })
 	log.info(`Appending quads to ${output}`)
 	fs.appendFileSync(output, serialized)
@@ -51,4 +47,3 @@ async function getQuads() {
 }
 
 void getQuads()
-

src/cli/repl/commands/parse.ts

@@ -1,27 +1,25 @@
-import type {
-	XmlBasedJson,
-	XmlParserConfig
-} from '../../../r-bridge'
+import type { XmlBasedJson} from '../../../r-bridge'
+import {childrenKey} from '../../../r-bridge'
+import {attributesKey, contentKey} from '../../../r-bridge'
 import {
-	DEFAULT_XML_PARSER_CONFIG,
-	getKeysGuarded, RawRType,
-	requestFromInput
+	parseCSV
 } from '../../../r-bridge'
+import {getKeysGuarded, RawRType, requestFromInput} from '../../../r-bridge'
 import {
 	extractLocation,
 	getTokenType,
 	objectWithArrUnwrap,
-	xlm2jsonObject
 } from '../../../r-bridge/lang-4.x/ast/parser/xml/internal'
 import type { OutputFormatter } from '../../../statistics'
 import { FontStyles } from '../../../statistics'
 import type { ReplCommand } from './main'
 import { SteppingSlicer } from '../../../core'
-import { deepMergeObject } from '../../../util/objects'
+import {csvToRecord} from '../../../r-bridge/lang-4.x/ast/parser/csv/format'
+import {convertToXmlBasedJson} from '../../../r-bridge/lang-4.x/ast/parser/csv/parser'
 
 type DepthList =  { depth: number, node: XmlBasedJson, leaf: boolean }[]
 
-function toDepthMap(xml: XmlBasedJson, config: XmlParserConfig): DepthList {
+function toDepthMap(xml: XmlBasedJson): DepthList {
 	const root = getKeysGuarded<XmlBasedJson>(xml, RawRType.ExpressionList)
 	const visit = [ { depth: 0, node: root } ]
 	const result: DepthList = []
@@ -32,7 +30,7 @@
 			continue
 		}
 
-		const children = current.node[config.childrenName] as XmlBasedJson[] | undefined ?? []
+		const children = current.node[childrenKey] as XmlBasedJson[] | undefined ?? []
 		result.push({ ...current, leaf: children.length === 0 })
 		children.reverse()
 
@@ -86,7 +84,7 @@
 	}
 }
 
-function depthListToTextTree(list: Readonly<DepthList>, config: XmlParserConfig, f: OutputFormatter): string {
+function depthListToTextTree(list: Readonly<DepthList>, f: OutputFormatter): string {
 	let result = ''
 
 	const deadDepths = new Set<number>()
@@ -100,14 +98,14 @@
 		result += f.reset()
 
 		const raw = objectWithArrUnwrap(node)
-		const content = raw[config.contentName] as string | undefined
-		const locationRaw = raw[config.attributeName] as XmlBasedJson | undefined
+		const content = raw[contentKey] as string | undefined
+		const locationRaw = raw[attributesKey] as XmlBasedJson | undefined
 		let location = ''
 		if(locationRaw !== undefined) {
 			location = retrieveLocationString(locationRaw)
 		}
 
-		const type = getTokenType(config.tokenMap, node)
+		const type = getTokenType(node)
 
 		if(leaf) {
 			const suffix = `${f.format(content ? JSON.stringify(content) : '', { style: FontStyles.Bold })}${f.format(location, { style: FontStyles.Italic })}`
@@ -134,9 +132,8 @@
 			request:        requestFromInput(remainingLine.trim())
 		}).allRemainingSteps()
 
-		const config = deepMergeObject<XmlParserConfig>(DEFAULT_XML_PARSER_CONFIG, { tokenMap: await shell.tokenMap() })
-		const object = xlm2jsonObject(config, result.parse)
+		const object = convertToXmlBasedJson(csvToRecord(parseCSV(result.parse)))
 
-		output.stdout(depthListToTextTree(toDepthMap(object, config), config, output.formatter))
+		output.stdout(depthListToTextTree(toDepthMap(object), output.formatter))
 	}
 }

src/cli/repl/server/connection.ts

@@ -1,7 +1,6 @@
 import type { StepResults} from '../../../core'
 import { LAST_STEP, printStepResult, SteppingSlicer, STEPS_PER_SLICE } from '../../../core'
-import type { NormalizedAst, RShell, XmlParserConfig } from '../../../r-bridge'
-import { DEFAULT_XML_PARSER_CONFIG } from '../../../r-bridge'
+import type { NormalizedAst, RShell } from '../../../r-bridge'
 import { sendMessage } from './send'
 import { answerForValidationError, validateBaseMessageFormat, validateMessage } from './validate'
 import type {
@@ -25,7 +24,6 @@ import {
 } from './messages/repl'
 import { replProcessAnswer } from '../core'
 import { ansiFormatter, voidFormatter } from '../../../statistics'
-import { deepMergeObject } from '../../../util/objects'
 import { LogLevel } from '../../../util/log'
 import type { ControlFlowInformation} from '../../../util/cfg/cfg'
 import { cfg2quads, extractCFG } from '../../../util/cfg/cfg'
@@ -132,7 +130,6 @@ export class FlowRServerConnection {
 		}
 
 		const config = (): QuadSerializationConfiguration => ({ context: message.filename ?? 'unknown', getId: defaultQuadIdGenerator() })
-		const parseConfig = deepMergeObject<XmlParserConfig>(DEFAULT_XML_PARSER_CONFIG, { tokenMap: await this.shell.tokenMap() })
 
 		if(message.format === 'n-quads') {
 			sendMessage<FileAnalysisResponseMessageNQuads>(this.socket, {
@@ -141,7 +138,7 @@ export class FlowRServerConnection {
 				id:      message.id,
 				cfg:     cfg ? cfg2quads(cfg, config()) : undefined,
 				results: {
-					parse:     await printStepResult('parse', results.parse as string, StepOutputFormat.RdfQuads, config(), parseConfig),
+					parse:     await printStepResult('parse', results.parse as string, StepOutputFormat.RdfQuads, config()),
 					normalize: await printStepResult('normalize', results.normalize as NormalizedAst, StepOutputFormat.RdfQuads, config()),
 					dataflow:  await printStepResult('dataflow', results.dataflow as DataflowInformation, StepOutputFormat.RdfQuads, config()),
 					ai:        ''
@@ -170,9 +167,8 @@ export class FlowRServerConnection {
 			shell:          this.shell,
 			// we have to make sure, that the content is not interpreted as a file path if it starts with 'file://' therefore, we do it manually
 			request:        {
-				request:                message.content === undefined ? 'file' : 'text',
-				content:                message.content ?? message.filepath as string,
-				ensurePackageInstalled: false
+				request: message.content === undefined ? 'file' : 'text',
+				content: message.content ?? message.filepath as string
 			},
 			criterion: [] // currently unknown
 		})

src/cli/statistics-helper-app.ts

@@ -57,7 +57,6 @@ if(options.compress) {
 const processedFeatures = new Set<FeatureKey>(options.features as FeatureKey[])
 
 const shell = new RShell()
-shell.tryToInjectHomeLibPath()
 
 initFileProvider(options['output-dir'])
 
@@ -111,4 +110,3 @@ async function getStatsForSingleFile() {
 }
 
 void getStatsForSingleFile()
-

src/core/print/parse-printer.ts

@@ -1,7 +1,10 @@
 import type { QuadSerializationConfiguration} from '../../util/quads'
 import { serialize2quads } from '../../util/quads'
-import { xlm2jsonObject } from '../../r-bridge/lang-4.x/ast/parser/xml/internal'
-import type { XmlBasedJson, XmlParserConfig } from '../../r-bridge'
+import type { XmlBasedJson} from '../../r-bridge'
+import {attributesKey, childrenKey, contentKey} from '../../r-bridge'
+import {parseCSV} from '../../r-bridge'
+import {csvToRecord} from '../../r-bridge/lang-4.x/ast/parser/csv/format'
+import {convertToXmlBasedJson} from '../../r-bridge/lang-4.x/ast/parser/csv/parser'
 
 function filterObject(obj: XmlBasedJson, keys: Set<string>): XmlBasedJson[] | XmlBasedJson {
 	if(typeof obj !== 'object') {
@@ -24,11 +27,11 @@
 
 }
 
-export function parseToQuads(code: string, config: QuadSerializationConfiguration, parseConfig: XmlParserConfig): string{
-	const obj = xlm2jsonObject(parseConfig, code)
+export function parseToQuads(code: string, config: QuadSerializationConfiguration): string{
+	const obj = convertToXmlBasedJson(csvToRecord(parseCSV(code)))
 	// recursively filter so that if the object contains one of the keys 'a', 'b' or 'c', all other keys are ignored
 	return serialize2quads(
-		filterObject(obj, new Set([parseConfig.attributeName, parseConfig.childrenName, parseConfig.contentName])) as XmlBasedJson,
+		filterObject(obj, new Set([attributesKey, childrenKey, contentKey])) as XmlBasedJson,
 		config
 	)
 }