-
-
Notifications
You must be signed in to change notification settings - Fork 67
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #244 from hackforla/dev
Updating master with latest dev
- Loading branch information
Showing
45 changed files
with
972 additions
and
1,046 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| *.json | ||
| *.test.js* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| import sqlalchemy as db | ||
| import pandas as pd | ||
| from sqlIngest import DataHandler | ||
| import databaseOrm | ||
|
|
||
|
|
||
| class DataCleaner(DataHandler): | ||
| def __init__(self, config=None, configFilePath=None, separator=','): | ||
| self.data = None | ||
| self.config = config | ||
| self.dbString = None if not self.config \ | ||
| else self.config['Database']['DB_CONNECTION_STRING'] | ||
| self.filePath = None | ||
| self.configFilePath = configFilePath | ||
| self.separator = separator | ||
| self.fields = databaseOrm.tableFields | ||
| self.insertParams = databaseOrm.insertFields | ||
| self.readParams = databaseOrm.readFields | ||
|
|
||
| def fetchData(self): | ||
| '''Retrieve data from mySql database instance''' | ||
| engine = db.create_engine(self.dbString) | ||
| self.data = pd.read_sql('ingest_staging_table', | ||
| con=engine, | ||
| index_col='srnumber') | ||
|
|
||
| def formatData(self): | ||
| '''Perform changes to data formatting to ensure compatibility | ||
| with cleaning and frontend processes''' | ||
| pass | ||
|
|
||
| def groupData(self): | ||
| '''Cluster data by geographic area to remove repeat instances | ||
| of 311 reports''' | ||
| pass | ||
|
|
||
| def cleaningReport(self): | ||
| '''Write out cleaning report summarizing operations performed | ||
| on data as well as data characteristics''' | ||
| pass | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| '''Class DataHandler workflow from initial load to SQL population''' | ||
| cleaner = DataCleaner() | ||
| cleaner.loadConfig(configFilePath='../settings.cfg') | ||
| cleaner.fetchData() | ||
| # can use inherited ingestData method to write to table | ||
| cleaner.ingestData(tableName='clean_data') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,83 +1,22 @@ | ||
| import React, { Component } from 'react'; | ||
| import axios from 'axios'; | ||
|
|
||
| import { getDataResources } from './Util/DataService'; | ||
| import { REQUESTS, COUNCILS } from './components/common/CONSTANTS'; | ||
| import React, { useEffect } from 'react'; | ||
| import { connect } from 'react-redux'; | ||
|
|
||
| import Header from './components/main/header/Header'; | ||
| import Body from './components/main/body/Body'; | ||
| import Footer from './components/main/footer/Footer'; | ||
|
|
||
| class App extends Component { | ||
| constructor() { | ||
| super(); | ||
|
|
||
| this.state = { | ||
| data: [], | ||
| year: '2015', | ||
| startMonth: '1', | ||
| endMonth: '12', | ||
| request: REQUESTS[0], | ||
| showMarkers: false, | ||
| showMarkersDropdown: true, | ||
| }; | ||
| } | ||
|
|
||
| componentDidMount() { | ||
| this.fetchData(); | ||
| } | ||
|
|
||
| updateState = (key, value, cb = () => null) => { | ||
| this.setState({ [key]: value }, () => { | ||
| this.fetchData(); // This is only for the dropdown component to fetch data on change | ||
| cb(); | ||
| }); | ||
| } | ||
|
|
||
| toggleShowMarkers = () => { | ||
| const { showMarkers } = this.state; | ||
| this.setState({ showMarkers: !showMarkers }); | ||
| } | ||
|
|
||
| fetchData = () => { | ||
| const dataUrl = this.buildDataUrl(); | ||
|
|
||
| axios.get(dataUrl) | ||
| .then(({ data }) => { | ||
| this.setState({ data }); | ||
| }) | ||
| .catch((error) => { | ||
| console.error(error); | ||
| }); | ||
| } | ||
|
|
||
| buildDataUrl = () => { | ||
| const { | ||
| startMonth, endMonth, year, request, | ||
| } = this.state; | ||
| const dataResources = getDataResources(); | ||
| return `https://data.lacity.org/resource/${dataResources[year]}.json?$select=location,zipcode,address,requesttype,status,ncname,streetname,housenumber&$where=date_extract_m(CreatedDate)+between+${startMonth}+and+${endMonth}+and+requesttype='${request}'`; | ||
| } | ||
|
|
||
| render() { | ||
| const { data, showMarkers, showMarkersDropdown } = this.state; | ||
|
|
||
| return ( | ||
| <div className="main"> | ||
| <Header | ||
| updateState={this.updateState} | ||
| toggleShowMarkers={this.toggleShowMarkers} | ||
| showMarkers={showMarkers} | ||
| showMarkersDropdown={showMarkersDropdown} | ||
| /> | ||
| <Body | ||
| data={data} | ||
| showMarkers={showMarkers} | ||
| /> | ||
| <Footer /> | ||
| </div> | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| export default App; | ||
| const App = () => { | ||
| useEffect(() => { | ||
| // fetch data on load?? | ||
| }, []); | ||
|
|
||
| return ( | ||
| <div className="main"> | ||
| <Header /> | ||
| <Body /> | ||
| <Footer /> | ||
| </div> | ||
| ); | ||
| }; | ||
|
|
||
| export default connect(null, null)(App); |
Oops, something went wrong.