Upload file to IPFS (#499)

Merging on behalf of the whole team

Co-authored-by: Dinek007 <[email protected]>
Co-authored-by: EmiM <[email protected]>
Co-authored-by: vinkabuki <[email protected]>

.eslintrc.js

@@ -26,6 +26,7 @@ module.exports = {
     '@typescript-eslint/no-dynamic-delete': 'off',
     '@typescript-eslint/default-param-last': 'off',
     '@typescript-eslint/restrict-plus-operands': 'off',
+    '@typescript-eslint/no-base-to-string': 'off',
     'generator-star-spacing': ['error', { before: false, after: true }],
     'yield-star-spacing': ['error', { before: false, after: true }],
     'react-hooks/exhaustive-deps': 'off',

packages/backend/src/socket/IOProxy.ts

@@ -24,7 +24,9 @@ import {
   NetworkData,
   ResponseCreateNetworkPayload,
   ErrorCodes,
-  AskForMessagesPayload
+  AskForMessagesPayload,
+  FileContent,
+  FileMetadata
 } from '@quiet/state-manager'
 import { emitError } from './errors'
 
@@ -77,6 +79,22 @@ export default class IOProxy {
     await this.getStorage(peerId).sendMessage(message)
   }
 
+  public uploadFile = async (peerId: string, file: FileContent) => {
+    await this.getStorage(peerId).uploadFile(file)
+  }
+
+  public uploadedFile = (metadata: FileMetadata) => {
+    this.io.emit(SocketActionTypes.UPLOADED_FILE, metadata)
+  }
+
+  public downloadFile = async (peerId: string, metadata: FileMetadata) => {
+    await this.getStorage(peerId).downloadFile(metadata)
+  }
+
+  public downloadedFile = (metadata: FileMetadata) => {
+    this.io.emit(SocketActionTypes.DOWNLOADED_FILE, metadata)
+  }
+
   // DMs
 
   public initializeConversation = async (

packages/backend/src/socket/listeners/connection.ts

@@ -8,7 +8,9 @@ import {
   SendMessagePayload,
   SocketActionTypes,
   SubscribeToTopicPayload,
-  AskForMessagesPayload
+  AskForMessagesPayload,
+  UploadFilePayload,
+  DownloadFilePayload
 } from '@quiet/state-manager'
 
 import IOProxy from '../IOProxy'
@@ -38,6 +40,18 @@ export const connections = (io, ioProxy: IOProxy) => {
         await ioProxy.sendMessage(payload.peerId, payload.message)
       }
     )
+    socket.on(
+      SocketActionTypes.UPLOAD_FILE,
+      async (payload: UploadFilePayload) => {
+        await ioProxy.uploadFile(payload.peerId, payload.file)
+      }
+    )
+    socket.on(
+      SocketActionTypes.DOWNLOAD_FILE,
+      async (payload: DownloadFilePayload) => {
+        await ioProxy.downloadFile(payload.peerId, payload.metadata)
+      }
+    )
     socket.on(
       SocketActionTypes.INITIALIZE_CONVERSATION,
       async (

packages/backend/src/storage/storage.test.ts

@@ -22,7 +22,10 @@ import {
   Store,
   Identity,
   ChannelMessage,
-  PublicChannelStorage
+  PublicChannel,
+  PublicChannelStorage,
+  FileContent,
+  FileMetadata
 } from '@quiet/state-manager'
 import { ConnectionsManager } from '../libp2p/connectionsManager'
 
@@ -289,3 +292,38 @@ describe('Message', () => {
     expect(spy).not.toHaveBeenCalled()
   })
 })
+
+describe.only('Files', () => {
+  it('is uploaded to IPFS then can be downloaded', async () => {
+    storage = new Storage(tmpAppDataPath, connectionsManager.ioProxy, community.id, { createPaths: false })
+
+    const peerId = await PeerId.create()
+    const libp2p = await createLibp2p(peerId)
+
+    await storage.init(libp2p, peerId)
+
+    await storage.initDatabases()
+
+    // Uploading
+    const uploadSpy = jest.spyOn(storage.io, 'uploadedFile')
+
+    const fileContent: FileContent = {
+      path: path.join(__dirname, '/testUtils/test-image.png'),
+      name: 'test-image',
+      ext: 'png'
+    }
+
+    await storage.uploadFile(fileContent)
+
+    expect(uploadSpy).toHaveBeenCalled()
+
+    // Downloading
+    const downloadSpy = jest.spyOn(storage.io, 'downloadedFile')
+
+    const uploadMetadata = uploadSpy.mock.calls[0][0]
+
+    await storage.downloadFile(uploadMetadata)
+
+    expect(downloadSpy).toHaveBeenCalled()
+  })
+})

packages/backend/src/storage/storage.ts

@@ -5,7 +5,13 @@ import {
   parseCertificate,
   verifyUserCert
 } from '@quiet/identity'
-import { ChannelMessage, PublicChannel, SaveCertificatePayload } from '@quiet/state-manager'
+import {
+  ChannelMessage,
+  PublicChannel,
+  SaveCertificatePayload,
+  FileContent,
+  FileMetadata
+} from '@quiet/state-manager'
 import * as IPFS from 'ipfs-core'
 import Libp2p from 'libp2p'
 import OrbitDB from 'orbit-db'
@@ -27,6 +33,8 @@ import { MessagesAccessController } from './MessagesAccessController'
 import logger from '../logger'
 import IOProxy from '../socket/IOProxy'
 import validate from '../validation/validators'
+import { CID } from 'multiformats/cid'
+import fs from 'fs'
 
 const log = logger('db')
 
@@ -58,7 +66,12 @@ export class Storage {
   public ipfsRepoPath: string
   private readonly communityId: string
 
-  constructor(quietDir: string, ioProxy: IOProxy, communityId: string, options?: Partial<StorageOptions>) {
+  constructor(
+    quietDir: string,
+    ioProxy: IOProxy,
+    communityId: string,
+    options?: Partial<StorageOptions>
+  ) {
     this.quietDir = quietDir
     this.io = ioProxy
     this.communityId = communityId
@@ -81,8 +94,11 @@ export class Storage {
 
     AccessControllers.addAccessController({ AccessController: MessagesAccessController })
 
-    // @ts-expect-error
-    this.orbitdb = await OrbitDB.createInstance(this.ipfs, { directory: this.orbitDbDir, AccessControllers: AccessControllers })
+    this.orbitdb = await OrbitDB.createInstance(this.ipfs, {
+      directory: this.orbitDbDir,
+      // @ts-expect-error
+      AccessControllers: AccessControllers
+    })
   }
 
   public async initDatabases() {
@@ -278,7 +294,7 @@ export class Storage {
           communityId: this.communityId
         })
       })
-      db.events.on('replicated', async (address) => {
+      db.events.on('replicated', async address => {
         log('Replicated.', address)
         const ids = this.getAllEventLogEntries<ChannelMessage>(db).map(msg => msg.id)
         this.io.sendMessagesIds({
@@ -301,10 +317,7 @@ export class Storage {
     log(`Subscribed to channel ${channel.address}`)
   }
 
-  public async askForMessages(
-    channelAddress: string,
-    ids: string[]
-  ) {
+  public async askForMessages(channelAddress: string, ids: string[]) {
     const repo = this.publicChannelsRepos.get(channelAddress)
     if (!repo) return
     const messages = this.getAllEventLogEntries<ChannelMessage>(repo.db)
@@ -325,12 +338,15 @@ export class Storage {
     }
     log(`Creating channel ${data.address}`)
 
-    const db: EventStore<ChannelMessage> = await this.orbitdb.log<ChannelMessage>(`channels.${data.address}`, {
-      accessController: {
-        type: 'messagesaccess',
-        write: ['*']
+    const db: EventStore<ChannelMessage> = await this.orbitdb.log<ChannelMessage>(
+      `channels.${data.address}`,
+      {
+        accessController: {
+          type: 'messagesaccess',
+          write: ['*']
+        }
       }
-    })
+    )
 
     const channel = this.channels.get(data.address)
     if (channel === undefined) {
@@ -358,7 +374,9 @@ export class Storage {
     }
     const repo = this.publicChannelsRepos.get(message.channelAddress)
     if (!repo) {
-      log.error(`Could not send message. No '${message.channelAddress}' channel in saved public channels`)
+      log.error(
+        `Could not send message. No '${message.channelAddress}' channel in saved public channels`
+      )
       return
     }
     try {
@@ -368,6 +386,71 @@ export class Storage {
     }
   }
 
+  public async uploadFile(fileContent: FileContent) {
+    log('uploadFile', fileContent)
+
+    let buffer: Buffer
+
+    try {
+      buffer = fs.readFileSync(fileContent.path)
+    } catch (e) {
+      log.error(`Couldn't open file ${fileContent.path}. Error: ${e.message}`)
+      throw new Error(`Couldn't open file ${fileContent.path}. Error: ${e.message}`)
+    }
+
+    // Create directory for file
+    const dirname = 'uploads'
+    await this.ipfs.files.mkdir(`/${dirname}`, { parents: true })
+    // Write file to IPFS
+    const uuid = `${Date.now()}_${Math.random().toString(36).substr(2.9)}`
+    const filename = `${uuid}_${fileContent.name}.${fileContent.ext}`
+    await this.ipfs.files.write(`/${dirname}/${filename}`, buffer, { create: true })
+    // Get uploaded file information
+    const entries = this.ipfs.files.ls(`/${dirname}`)
+    for await (const entry of entries) {
+      if (entry.name === filename) {
+        const metadata: FileMetadata = {
+          ...fileContent,
+          path: fileContent.path,
+          cid: entry.cid.toString()
+        }
+        this.io.uploadedFile(metadata)
+        break
+      }
+    }
+  }
+
+  public async downloadFile(metadata: FileMetadata) {
+    const _CID = CID.parse(metadata.cid)
+    const entries = this.ipfs.cat(_CID)
+
+    const downloadDirectory = path.join(this.quietDir, 'downloads', metadata.cid)
+    createPaths([downloadDirectory])
+
+    const fileName = metadata.name + metadata.ext
+    const filePath = `${path.join(downloadDirectory, fileName)}`
+
+    const writeStream = fs.createWriteStream(filePath)
+
+    for await (const entry of entries) {
+      await new Promise<void>((resolve, reject) => {
+        writeStream.write(entry, err => {
+          if (err) reject(err)
+          resolve()
+        })
+      })
+    }
+
+    writeStream.end()
+
+    const fileMetadata: FileMetadata = {
+      ...metadata,
+      path: filePath
+    }
+
+    this.io.downloadedFile(fileMetadata)
+  }
+
   public async initializeConversation(address: string, encryptedPhrase: string): Promise<void> {
     if (!validate.isConversation(address, encryptedPhrase)) {
       log.error('STORAGE: Invalid conversation format')
@@ -479,7 +562,10 @@ export class Storage {
       log('Certificate is either null or undefined, not saving to db')
       return false
     }
-    const verification = await verifyUserCert(payload.rootPermsData.certificate, payload.certificate)
+    const verification = await verifyUserCert(
+      payload.rootPermsData.certificate,
+      payload.certificate
+    )
     if (verification.resultCode !== 0) {
       log.error('Certificate is not valid')
       log.error(verification.resultMessage)

packages/backend/src/validation/validators.ts

@@ -1,11 +1,23 @@
 import _ from 'validator'
 import joi from 'joi'
+import logger from '../logger'
 import { ChannelMessage, PublicChannel } from '@quiet/state-manager'
+const log = logger('validators')
 
 const messageSchema = joi.object({
   id: joi.string().required(),
   type: joi.number().required().positive().integer(),
-  message: joi.string().required(),
+  message: joi.alternatives(joi.string(), joi.binary()).required(),
+  media: joi.object({
+    path: joi.string().allow(null),
+    name: joi.string().required(),
+    ext: joi.string().required(),
+    cid: joi.string().required(),
+    message: joi.object({
+      id: joi.string().required(),
+      channelAddress: joi.string().required()
+    })
+  }),
   createdAt: joi.number().required(),
   channelAddress: joi.string().required(),
   signature: joi.string().required(),
@@ -43,6 +55,7 @@ export const isDirectMessage = (msg: string): boolean => {
 
 export const isMessage = (msg: ChannelMessage): boolean => {
   const value = messageSchema.validate(msg)
+  if (value.error) log.error('isMessage', value.error)
   return !value.error
 }
 

packages/desktop/.gitignore

@@ -51,3 +51,6 @@ src/renderer/zcash/index.node
 
 /cypress/snapshots/actual
 /cypress/snapshots/diff
+
+# Tor
+/tor/libevent-*

packages/desktop/src/main/files.ts

@@ -0,0 +1,14 @@
+import path from 'path'
+
+export const openFiles = (paths: string[]) => {
+  const data = {}
+  paths.forEach((filePath: string) => {
+    const id = `${Date.now()}_${Math.random().toString(36).substring(0, 20)}`
+    data[id] = {
+      path: filePath,
+      name: path.basename(filePath, path.extname(filePath)),
+      ext: path.extname(filePath)
+    }
+  })
+  return data
+}