SNOWFLAKE storage backend support

.env.example

@@ -65,6 +65,7 @@ DEKART_UX_NOT_FOUND_ERROR_INFO_HTML=
 DEKART_UX_SAMPLE_QUERY_SQL=
 
 #dev
+DEKART_STREAM_TIMEOUT=
 DEKART_LOG_DEBUG=1
 DEKART_LOG_PRETTY=1
 DEKART_STATIC_FILES=

src/client/DatasetSettingsModal.js

@@ -9,7 +9,7 @@ import { closeDatasetSettingsModal, removeDataset, updateDatasetName } from './a
 
 function ModalFooter ({ saving, setSaving, name, datasetId }) {
   const dispatch = useDispatch()
-  const numDatasets = useSelector(state => state.datasets.length)
+  const numDatasets = useSelector(state => state.dataset.list.length)
 
   return (
     <div className={styles.modalFooter}>
@@ -45,11 +45,11 @@ function ModalFooter ({ saving, setSaving, name, datasetId }) {
 }
 
 export default function DatasetSettingsModal () {
-  const datasetId = useSelector(state => state.datasetSettings.datasetId)
-  const dataset = useSelector(state => state.datasets.find(d => d.id === datasetId))
+  const datasetId = useSelector(state => state.dataset.settings.datasetId)
+  const dataset = useSelector(state => state.dataset.list.find(d => d.id === datasetId))
   const queries = useSelector(state => state.queries)
   const files = useSelector(state => state.files)
-  const visible = useSelector(state => state.datasetSettings.visible)
+  const visible = useSelector(state => state.dataset.settings.visible)
 
   const dispatch = useDispatch()
 

src/client/Downloading.js

@@ -4,7 +4,7 @@ import message from 'antd/es/message'
 import { useEffect } from 'react'
 
 function DownloadingMessage () {
-  const downloadingDatasets = useSelector(state => state.downloadingDatasets)
+  const downloadingDatasets = useSelector(state => state.dataset.downloading)
   const files = useSelector(state => state.files)
   const queries = useSelector(state => state.queries)
   const size = downloadingDatasets.reduce((size, { queryId, fileId }) => {
@@ -26,7 +26,7 @@ function DownloadingMessage () {
 
 let hideDownloading = null
 export default function Downloading () {
-  const downloadingDatasets = useSelector(state => state.downloadingDatasets)
+  const downloadingDatasets = useSelector(state => state.dataset.downloading)
   const show = downloadingDatasets.length > 0
   const [api, contextHolder] = message.useMessage()
   useEffect(() => {

src/client/ReportPage.js

@@ -111,10 +111,10 @@ function getTabPane (dataset, queries, files, status) {
 }
 
 function DatasetSection ({ reportId }) {
-  const datasets = useSelector(state => state.datasets)
+  const datasets = useSelector(state => state.dataset.list)
   const queries = useSelector(state => state.queries)
   const files = useSelector(state => state.files)
-  const activeDataset = useSelector(state => state.activeDataset)
+  const activeDataset = useSelector(state => state.dataset.active)
   const report = useSelector(state => state.report)
   const queryStatus = useSelector(state => state.queryStatus)
   const { canWrite } = report

src/client/actions/dataset.js

@@ -1,12 +1,13 @@
 import { CreateDatasetRequest, RemoveDatasetRequest, UpdateDatasetConnectionRequest, UpdateDatasetNameRequest } from '../../proto/dekart_pb'
 import { Dekart } from '../../proto/dekart_pb_service'
 import { grpcCall } from './grpc'
-import { downloading, setError, finishDownloading, success } from './message'
+import { downloading, setError, finishDownloading, success, info, warn } from './message'
 import { addDataToMap, toggleSidePanel, removeDataset as removeDatasetFromKepler, reorderLayer } from '@dekart-xyz/kepler.gl/dist/actions'
 import { processCsvData, processGeojson } from '@dekart-xyz/kepler.gl/dist/processors'
 import { get } from '../lib/api'
 import { KeplerGlSchema } from '@dekart-xyz/kepler.gl/dist/schemas'
 import getDatasetName from '../lib/getDatasetName'
+import { runQuery } from './query'
 
 export function createDataset (reportId) {
   return (dispatch, getState) => {
@@ -19,7 +20,7 @@ export function createDataset (reportId) {
 
 export function setActiveDataset (datasetId) {
   return (dispatch, getState) => {
-    const { datasets } = getState()
+    const { list: datasets } = getState().dataset
     const dataset = datasets.find(d => d.id === datasetId) || datasets[0]
     if (dataset) {
       dispatch({ type: setActiveDataset.name, dataset })
@@ -29,7 +30,7 @@ export function setActiveDataset (datasetId) {
 
 export function updateDatasetName (datasetId, name) {
   return async (dispatch, getState) => {
-    const { datasets } = getState()
+    const { list: datasets } = getState().dataset
     const dataset = datasets.find(d => d.id === datasetId)
     if (!dataset) {
       return
@@ -44,7 +45,7 @@ export function updateDatasetName (datasetId, name) {
 
 export function updateDatasetConnection (datasetId, connectionId) {
   return async (dispatch, getState) => {
-    const { datasets } = getState()
+    const { list: datasets } = getState().dataset
     const dataset = datasets.find(d => d.id === datasetId)
     if (!dataset) {
       return
@@ -59,7 +60,7 @@ export function updateDatasetConnection (datasetId, connectionId) {
 
 export function removeDataset (datasetId) {
   return async (dispatch, getState) => {
-    const { datasets, activeDataset } = getState()
+    const { list: datasets, active: activeDataset } = getState().dataset
     if (activeDataset.id === datasetId) {
       // removed active query
       const datasetsLeft = datasets.filter(q => q.id !== datasetId)
@@ -81,6 +82,8 @@ export function removeDataset (datasetId) {
 
 export function downloadDataset (dataset, sourceId, extension, prevDatasetsList) {
   return async (dispatch, getState) => {
+    const { dataset: { list: datasets }, files, queries, keplerGl } = getState()
+    const label = getDatasetName(dataset, queries, files)
     dispatch({ type: downloadDataset.name, dataset })
     dispatch(downloading(dataset))
     const { token } = getState()
@@ -95,11 +98,20 @@ export function downloadDataset (dataset, sourceId, extension, prevDatasetsList)
         data = processGeojson(json)
       }
     } catch (err) {
-      dispatch(setError(err))
+      dispatch(finishDownloading(dataset)) // remove downloading message
+      if (err.status === 410 && dataset.queryId) { // gone from dw query temporary storage
+        const { canRun, queryText } = getState().queryStatus[dataset.queryId]
+        if (!canRun) {
+          dispatch(warn(<><i>{label}</i> result expired</>, false))
+          return
+        }
+        dispatch(info(<><i>{label}</i> result expired, re-running</>))
+        dispatch(runQuery(dataset.queryId, queryText))
+      } else {
+        dispatch(setError(err))
+      }
       return
     }
-    const { datasets, files, queries, keplerGl } = getState()
-    const label = getDatasetName(dataset, queries, files)
     // check if dataset was already added to kepler
     const addedDatasets = getState().keplerGl.kepler.visState.datasets
     const prevDataset = prevDatasetsList.find(d => d.id in addedDatasets)

src/client/actions/message.js

@@ -10,10 +10,25 @@ export function downloading (dataset) {
   return { type: downloading.name, dataset }
 }
 
-export function finishDownloading (dataset) {
-  return { type: finishDownloading.name, dataset }
+export function finishDownloading (dataset, gone = false) {
+  return { type: finishDownloading.name, dataset, gone }
 }
 
+export function warn (content, transitive = true) {
+  if (!transitive) {
+    message.warn({
+      content,
+      duration: 10000,
+      style
+    })
+  } else {
+    message.warn({
+      content,
+      style
+    })
+  }
+  return { type: warn.name }
+}
 export function success (content) {
   message.success({
     content,

src/client/actions/report.js

@@ -83,7 +83,7 @@ function shouldDownloadQueryText (query, prevQueriesList, queriesList) {
 export function reportUpdate (reportStreamResponse) {
   const { report, queriesList, datasetsList, filesList } = reportStreamResponse
   return async (dispatch, getState) => {
-    const { queries: prevQueriesList, datasets: prevDatasetsList, report: prevReport, files: prevFileList, env, connection } = getState()
+    const { queries: prevQueriesList, dataset: { list: prevDatasetsList }, report: prevReport, files: prevFileList, env, connection } = getState()
     dispatch({
       type: reportUpdate.name,
       report,
@@ -126,7 +126,7 @@ export function reportUpdate (reportStreamResponse) {
       let extension = 'csv'
       if (dataset.queryId) {
         const query = queriesList.find(q => q.id === dataset.queryId)
-        if (shouldAddQuery(query, prevQueriesList, queriesList) || shouldUpdateDataset(dataset, prevDatasetsList)) {
+        if (shouldAddQuery(query, prevQueriesList) || shouldUpdateDataset(dataset, prevDatasetsList)) {
           dispatch(downloadDataset(
             dataset,
             query.jobResultId,

src/client/lib/shouldAddQuery.js

@@ -1,4 +1,4 @@
-export function shouldAddQuery (query, prevQueriesList, queriesList) {
+export function shouldAddQuery (query, prevQueriesList) {
   if (!query.jobResultId) {
     return false
   }

src/client/reducers/connectionReducer.js

@@ -107,7 +107,7 @@ function userDefined (state = false, action) {
   switch (action.type) {
     case setEnv.name: {
       const { BIGQUERY_PROJECT_ID, CLOUD_STORAGE_BUCKET, DATASOURCE } = action.variables
-      return (BIGQUERY_PROJECT_ID === '' && DATASOURCE === 'BQ') || CLOUD_STORAGE_BUCKET === ''
+      return (BIGQUERY_PROJECT_ID === '' && DATASOURCE === 'BQ') || (CLOUD_STORAGE_BUCKET === '' && DATASOURCE !== 'SNOWFLAKE')
     }
     default:
       return state

src/client/reducers/datasetReducer.js

@@ -0,0 +1,73 @@
+import { combineReducers } from 'redux'
+import { closeDatasetSettingsModal, openDatasetSettingsModal, setActiveDataset } from '../actions/dataset'
+import { downloading as downloadingAction, finishDownloading } from '../actions/message'
+import { openReport, reportUpdate } from '../actions/report'
+
+function downloading (state = [], action) {
+  const { dataset } = action
+  switch (action.type) {
+    case downloadingAction.name:
+      return state.concat(dataset)
+    case finishDownloading.name:
+      return state.filter(d => d.id !== dataset.id)
+    default:
+      return state
+  }
+}
+
+function active (state = null, action) {
+  const { datasetsList, prevDatasetsList } = action
+  switch (action.type) {
+    case openReport.name:
+      return null
+    case setActiveDataset.name:
+      return action.dataset
+    case reportUpdate.name:
+      if (!state) {
+        return datasetsList[0] || state
+      }
+      if (datasetsList.length > prevDatasetsList.length) {
+        return datasetsList.slice(-1)[0]
+      }
+      return {
+        ...(datasetsList.find(d => d.id === state.id) || datasetsList[0])
+      }
+    default:
+      return state
+  }
+}
+
+function list (state = [], action) {
+  switch (action.type) {
+    case openReport.name:
+      return []
+    case reportUpdate.name:
+      return action.datasetsList
+    default:
+      return state
+  }
+}
+
+function settings (state = { datasetId: null, visible: false }, action) {
+  switch (action.type) {
+    case openDatasetSettingsModal.name:
+      return {
+        datasetId: action.datasetId,
+        visible: true
+      }
+    case closeDatasetSettingsModal.name:
+      return {
+        datasetId: null,
+        visible: false
+      }
+    default:
+      return state
+  }
+}
+
+export default combineReducers({
+  downloading,
+  active,
+  settings,
+  list
+})