Skip to content

Commit

Permalink
feat!: add data, cache and temp dirs to FileSystem (#1306)
Browse files Browse the repository at this point in the history 
Signed-off-by: Ariel Gentile <[email protected]>

BREAKING CHANGE:

Agent-produced files will now be divided in different system paths depending on their nature: data, temp and cache. Previously, they were located at a single location, defaulting to a temporary directory.

If you specified a custom path in `FileSystem` object constructor, you now must provide an object containing `baseDataPath`, `baseTempPath` and `baseCachePath`. They can point to the same path, although it's recommended to specify different path to avoid future file clashes.
  • Loading branch information
@genaris
genaris authored Feb 18, 2023
1 parent af384e8 commit ff5596d
Show file tree
Hide file tree
Showing 16 changed files with 91 additions and 81 deletions.
6 changes: 3 additions & 3 deletions packages/anoncreds/src/utils/tails.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ import type { AgentContext, FileSystem } from '@aries-framework/core'

import { TypedArrayEncoder, InjectionSymbols } from '@aries-framework/core'

const getTailsFilePath = (basePath: string, tailsHash: string) => `${basePath}/afj/anoncreds/tails/${tailsHash}`
const getTailsFilePath = (cachePath: string, tailsHash: string) => `${cachePath}/anoncreds/tails/${tailsHash}`

export function tailsFileExists(agentContext: AgentContext, tailsHash: string): Promise<boolean> {
const fileSystem = agentContext.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
const tailsFilePath = getTailsFilePath(fileSystem.basePath, tailsHash)
const tailsFilePath = getTailsFilePath(fileSystem.cachePath, tailsHash)

return fileSystem.exists(tailsFilePath)
}
Expand All @@ -27,7 +27,7 @@ export async function downloadTailsFile(

// hash is used as file identifier
const tailsExists = await tailsFileExists(agentContext, tailsHashBase58)
const tailsFilePath = getTailsFilePath(fileSystem.basePath, tailsHashBase58)
const tailsFilePath = getTailsFilePath(fileSystem.cachePath, tailsHashBase58)
agentContext.config.logger.debug(
`Tails file for ${tailsLocation} ${tailsExists ? 'is stored' : 'is not stored'} at ${tailsFilePath}`
)
Expand Down
13 changes: 11 additions & 2 deletions packages/askar/src/utils/askarWalletConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,16 @@ export const keyDerivationMethodToStoreKeyMethod = (keyDerivationMethod?: KeyDer
return correspondenceTable[keyDerivationMethod] as StoreKeyMethod
}

export const uriFromWalletConfig = (walletConfig: WalletConfig, basePath: string): { uri: string; path?: string } => {
/**
* Creates a proper askar wallet URI value based on walletConfig
* @param walletConfig WalletConfig object
* @param afjDataPath framework data path (used in case walletConfig.storage.path is undefined)
* @returns string containing the askar wallet URI
*/
export const uriFromWalletConfig = (
walletConfig: WalletConfig,
afjDataPath: string
): { uri: string; path?: string } => {
let uri = ''
let path

Expand All @@ -31,7 +40,7 @@ export const uriFromWalletConfig = (walletConfig: WalletConfig, basePath: string
if (walletConfig.storage.inMemory) {
uri = 'sqlite://:memory:'
} else {
path = `${(walletConfig.storage.path as string) ?? basePath + '/wallet'}/${walletConfig.id}/sqlite.db`
path = (walletConfig.storage.path as string) ?? `${afjDataPath}/wallet/${walletConfig.id}/sqlite.db`
uri = `sqlite://${path}`
}
} else if (walletConfig.storage.type === 'postgres') {
Expand Down
4 changes: 2 additions & 2 deletions packages/askar/src/wallet/AskarWallet.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ export class AskarWallet implements Wallet {
}

try {
const { uri } = uriFromWalletConfig(this.walletConfig, this.fileSystem.basePath)
const { uri } = uriFromWalletConfig(this.walletConfig, this.fileSystem.dataPath)
await Store.remove(uri)
} catch (error) {
const errorMessage = `Error deleting wallet '${this.walletConfig.id}': ${error.message}`
Expand Down Expand Up @@ -689,7 +689,7 @@ export class AskarWallet implements Wallet {
}

private async getAskarWalletConfig(walletConfig: WalletConfig) {
const { uri, path } = uriFromWalletConfig(walletConfig, this.fileSystem.basePath)
const { uri, path } = uriFromWalletConfig(walletConfig, this.fileSystem.dataPath)

// Make sure path exists before creating the wallet
if (path) {
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/modules/indy/services/IndyUtilitiesService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ export class IndyUtilitiesService {
public async downloadTails(hash: string, tailsLocation: string): Promise<BlobReaderHandle> {
try {
this.logger.debug(`Checking to see if tails file for URL ${tailsLocation} has been stored in the FileSystem`)
const filePath = `${this.fileSystem.basePath}/afj/tails/${hash}`
const filePath = `${this.fileSystem.cachePath}/tails/${hash}`

const tailsExists = await this.fileSystem.exists(filePath)
this.logger.debug(`Tails file for ${tailsLocation} ${tailsExists ? 'is stored' : 'is not stored'} at ${filePath}`)
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/modules/ledger/IndyPool.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ export class IndyPool {
if (this.poolConfig.genesisPath) return this.poolConfig.genesisPath

// Determine the genesisPath
const genesisPath = this.fileSystem.basePath + `/afj/genesis-${this.poolConfig.id}.txn`
const genesisPath = this.fileSystem.tempPath + `/genesis-${this.poolConfig.id}.txn`
// Store genesis data if provided
if (this.poolConfig.genesisTransactions) {
await this.fileSystem.write(genesisPath, this.poolConfig.genesisTransactions)
Expand Down
5 changes: 4 additions & 1 deletion packages/core/src/storage/FileSystem.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@ export interface DownloadToFileOptions {
}

export interface FileSystem {
readonly basePath: string
readonly dataPath: string
readonly cachePath: string
readonly tempPath: string

exists(path: string): Promise<boolean>
createDirectory(path: string): Promise<void>
write(path: string, data: string): Promise<void>
read(path: string): Promise<string>
delete(path: string): Promise<void>
downloadToFile(url: string, path: string, options?: DownloadToFileOptions): Promise<void>
}
7 changes: 6 additions & 1 deletion packages/core/src/storage/migration/UpdateAssistant.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,13 +157,18 @@ export class UpdateAssistant<Agent extends BaseAgent<any> = BaseAgent> {
`Successfully updated agent storage from version ${update.fromVersion} to version ${update.toVersion}`
)
}
// Delete backup file, as it is not needed anymore
await this.fileSystem.delete(this.getBackupPath(updateIdentifier))
} catch (error) {
this.agent.config.logger.fatal('An error occurred while updating the wallet. Restoring backup', {
error,
})
// In the case of an error we want to restore the backup
await this.restoreBackup(updateIdentifier)

// Delete backup file, as wallet was already restored (backup-error file will persist though)
await this.fileSystem.delete(this.getBackupPath(updateIdentifier))

throw error
}
} catch (error) {
Expand Down Expand Up @@ -192,7 +197,7 @@ export class UpdateAssistant<Agent extends BaseAgent<any> = BaseAgent> {
}

private getBackupPath(backupIdentifier: string) {
return `${this.fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
return `${this.fileSystem.dataPath}/migration/backup/${backupIdentifier}`
}

private async createBackup(backupIdentifier: string) {
Expand Down
24 changes: 0 additions & 24 deletions packages/core/src/storage/migration/__tests__/0.1.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
dependencyManager
)

const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)

const updateAssistant = new UpdateAssistant(agent, {
v0_1ToV0_2: {
mediationRoleUpdateStrategy,
Expand Down Expand Up @@ -79,10 +77,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
delete storageService.records.MEDIATOR_ROUTING_RECORD
expect(storageService.records).toMatchSnapshot(mediationRoleUpdateStrategy)

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()
}
Expand Down Expand Up @@ -110,8 +104,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
dependencyManager
)

const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)

const updateAssistant = new UpdateAssistant(agent, {
v0_1ToV0_2: {
mediationRoleUpdateStrategy: 'doNotChange',
Expand Down Expand Up @@ -142,10 +134,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
delete storageService.records.MEDIATOR_ROUTING_RECORD
expect(storageService.records).toMatchSnapshot()

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()

Expand Down Expand Up @@ -174,8 +162,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
dependencyManager
)

const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)

const updateAssistant = new UpdateAssistant(agent, {
v0_1ToV0_2: {
mediationRoleUpdateStrategy: 'doNotChange',
Expand Down Expand Up @@ -206,10 +192,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
delete storageService.records.MEDIATOR_ROUTING_RECORD
expect(storageService.records).toMatchSnapshot()

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()

Expand Down Expand Up @@ -242,8 +224,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
dependencyManager
)

const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)

const updateAssistant = new UpdateAssistant(agent, {
v0_1ToV0_2: {
mediationRoleUpdateStrategy: 'doNotChange',
Expand Down Expand Up @@ -274,10 +254,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
delete storageService.records.MEDIATOR_ROUTING_RECORD
expect(storageService.records).toMatchSnapshot()

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()

Expand Down
19 changes: 0 additions & 19 deletions packages/core/src/storage/migration/__tests__/0.2.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import { UpdateAssistant } from '../UpdateAssistant'

const backupDate = new Date('2022-01-21T22:50:20.522Z')
jest.useFakeTimers().setSystemTime(backupDate)
const backupIdentifier = backupDate.getTime()

const walletConfig = {
id: `Wallet: 0.2 Update`,
Expand Down Expand Up @@ -46,8 +45,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
dependencyManager
)

const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)

const updateAssistant = new UpdateAssistant(agent, {
v0_1ToV0_2: {
mediationRoleUpdateStrategy: 'doNotChange',
Expand Down Expand Up @@ -83,10 +80,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
delete storageService.records.MEDIATOR_ROUTING_RECORD
expect(storageService.records).toMatchSnapshot()

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()

Expand Down Expand Up @@ -119,8 +112,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
dependencyManager
)

const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)

// We need to manually initialize the wallet as we're using the in memory wallet service
// When we call agent.initialize() it will create the wallet and store the current framework
// version in the in memory storage service. We need to manually set the records between initializing
Expand All @@ -137,10 +128,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
delete storageService.records.MEDIATOR_ROUTING_RECORD
expect(storageService.records).toMatchSnapshot()

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()

Expand Down Expand Up @@ -170,8 +157,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
dependencyManager
)

const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)

// We need to manually initialize the wallet as we're using the in memory wallet service
// When we call agent.initialize() it will create the wallet and store the current framework
// version in the in memory storage service. We need to manually set the records between initializing
Expand All @@ -189,10 +174,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {

expect(storageService.records).toMatchSnapshot()

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()

Expand Down
4 changes: 0 additions & 4 deletions packages/core/src/storage/migration/__tests__/0.3.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,6 @@ describe('UpdateAssistant | v0.3 - v0.3.1', () => {
delete storageService.records.MEDIATOR_ROUTING_RECORD
expect(storageService.records).toMatchSnapshot()

// Need to remove backupFiles after each run so we don't get IOErrors
const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
unlinkSync(backupPath)

await agent.shutdown()
await agent.wallet.delete()

Expand Down
15 changes: 10 additions & 5 deletions packages/core/src/storage/migration/__tests__/backup.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ describe('UpdateAssistant | Backup', () => {
beforeEach(async () => {
agent = new Agent(agentOptions)
const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
backupPath = `${fileSystem.dataPath}/migration/backup/${backupIdentifier}`

// If tests fail it's possible the cleanup has been skipped. So remove before running tests
const doesFileSystemExist = await fileSystem.exists(backupPath)
Expand Down Expand Up @@ -85,11 +85,16 @@ describe('UpdateAssistant | Backup', () => {
// Backup should not exist before update
expect(await fileSystem.exists(backupPath)).toBe(false)

const walletSpy = jest.spyOn(agent.wallet, 'export')

// Create update
await updateAssistant.update()

// Backup should exist after update
expect(await fileSystem.exists(backupPath)).toBe(true)
// A wallet export should have been initiated
expect(walletSpy).toHaveBeenCalledWith({ key: agent.wallet.walletConfig?.key, path: backupPath })

// Backup should be cleaned after update
expect(await fileSystem.exists(backupPath)).toBe(false)

expect(
(await credentialRepository.getAll(agent.context)).sort((a, b) => a.id.localeCompare(b.id))
Expand Down Expand Up @@ -142,8 +147,8 @@ describe('UpdateAssistant | Backup', () => {

expect(updateError?.cause?.message).toEqual("Uh oh I'm broken")

// Backup should exist after update
expect(await fileSystem.exists(backupPath)).toBe(true)
// Only backup error should exist after update
expect(await fileSystem.exists(backupPath)).toBe(false)
expect(await fileSystem.exists(`${backupPath}-error`)).toBe(true)

// Wallet should be same as when we started because of backup
Expand Down
2 changes: 1 addition & 1 deletion packages/indy-sdk/src/ledger/IndySdkPool.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ export class IndySdkPool {
if (this.poolConfig.genesisPath) return this.poolConfig.genesisPath

// Determine the genesisPath
const genesisPath = this.fileSystem.basePath + `/afj/genesis-${this.poolConfig.id}.txn`
const genesisPath = this.fileSystem.tempPath + `/genesis-${this.poolConfig.id}.txn`
// Store genesis data if provided
if (this.poolConfig.genesisTransactions) {
await this.fileSystem.write(genesisPath, this.poolConfig.genesisTransactions)
Expand Down
Loading

0 comments on commit ff5596d

Please sign in to comment.