Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CSV summary Addition to production #178

Merged
merged 12 commits into from
Jun 27, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
124 summary csv files (#175)
* aAdding summary csv reports

* adding comments
feydan authored Jun 27, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit ca1a46b75690ba642122c7a4f8c39844593798d6
1 change: 1 addition & 0 deletions frontend/.gitignore
Original file line number Diff line number Diff line change
@@ -23,3 +23,4 @@ pnpm-debug.log*
*.sln
*.sw?
.log
package-lock.json
5 changes: 5 additions & 0 deletions frontend/jest-puppeteer.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module.exports = {
globalSetup: './setup.js',
globalTeardown: './teardown.js',
testEnvironment: './puppeteer_environment.js',
}
8 changes: 4 additions & 4 deletions frontend/jest.config.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
globalSetup: './setup.js',
globalTeardown: './teardown.js',
testEnvironment: './puppeteer_environment.js',
}
preset: 'ts-jest',
testEnvironment: 'node',
};
18 changes: 12 additions & 6 deletions frontend/package.json
Original file line number Diff line number Diff line change
@@ -7,24 +7,22 @@
"build": "vite build",
"lint": "vue-cli-service lint",
"dev": "vite",
"test": "jest"
"test": "jest -c jest-puppeteer.config.js",
"test-ts": "jest"
},
"dependencies": {
"@fawmi/vue-google-maps": "^0.9.72",
"@googlemaps/js-api-loader": "^1.14.3",
"@mdi/font": "^7.0.96",
"@tensorflow/tfjs": "^3.18.0",
"@types/crypto-js": "^4.1.1",
"@types/file-saver": "^2.0.5",
"@types/google.maps": "^3.49.2",
"@vue/tsconfig": "^0.1.3",
"axios": "^0.27.2",
"core-js": "^3.23.5",
"crypto-js": "^4.1.1",
"dexie": "^3.2.2",
"file-saver": "^2.0.5",
"fuse.js": "^6.6.2",
"jest": "^29.5.0",
"json-schema": "^0.4.0",
"jszip": "^3.10.0",
"loglevel": "^1.8.0",
"p-queue": "^7.3.0",
@@ -42,10 +40,16 @@
"vue-inner-image-zoom": "^2.0.0",
"vue-lodash": "^2.1.2",
"vue-router": "^4.1.2",
"vuetify": "3.0.0-beta.5",
"vuetify": "^3.3.5",
"webfontloader": "^1.6.28"
},
"devDependencies": {
"@types/crypto-js": "^4.1.1",
"@types/file-saver": "^2.0.5",
"@types/google.maps": "^3.49.2",
"@types/jest": "^29.5.2",
"@types/json-schema": "^7.0.12",
"@types/lodash": "^4.14.195",
"@types/webfontloader": "^1.6.34",
"@typescript-eslint/eslint-plugin": "^5.30.7",
"@typescript-eslint/parser": "^5.30.7",
@@ -60,9 +64,11 @@
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.2.1",
"eslint-plugin-vue": "^9.2.0",
"jest": "^29.5.0",
"jest-puppeteer": "^9.0.0",
"prettier": "^2.7.1",
"sass": "^1.53.0",
"ts-jest": "^29.1.0",
"typescript": "^4.7.4",
"vite": "^3.2.7",
"webpack-plugin-vuetify": "^2.0.0-alpha.0"
39 changes: 39 additions & 0 deletions frontend/src/lib/csv/csv.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { JSONSchema7Definition } from 'json-schema'
import {
flatMapPropertiesRecursive,
getObjOrArrProperties,
} from './schema-parser'
import { MapJsonProperties } from './types'

export const objectToCsv =
<T>(
headers: string[],
fnFlatten: (obj: T) => (string | number | boolean)[][],
joinStr: string = ',',
) =>
(obj: T) => {
const csvData = fnFlatten(obj)
const errorRows = csvData.filter((v) => v.length !== headers.length)
if (errorRows.length !== 0) {
throw new Error(
`All csv rows must have the same length as the headers array ${JSON.stringify(
headers,
)}, see incorrect rows: ${JSON.stringify(errorRows)}`,
)
}
return [headers, ...csvData].map((row) => row.join(joinStr)).join('\n')
}

/**
* Traverses a json schema object and returns a flat array
* representing the keys of all objects and subobjects.
* Maintains nesting representation by joining keys together with '.'
*/
export const getCsvHeadersFromJsonSchema = (
jsonSchema: JSONSchema7Definition,
) => {
const props = getObjOrArrProperties(jsonSchema)
return props ? flatMapPropertiesRecursive(getJsonKeys, '.')(props) : []
}

const getJsonKeys: MapJsonProperties<string> = (k) => [k]
174 changes: 174 additions & 0 deletions frontend/src/lib/csv/schema-parser.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
import { describe, expect, test } from '@jest/globals'
import { JSONSchema7 } from 'json-schema'
import { getCsvHeadersFromJsonSchema, objectToCsv } from './csv'

describe('csv', () => {
const headers = ['id', 'fruit', 'cultivar', 'inSeason']
const obj = {
id: 1,
fruits: [
{
fruit: 'apple',
cultivar: 'gala',
inSeason: false,
},
{
fruit: 'apple',
cultivar: 'red delicious',
inSeason: true,
},
{
fruit: 'pear',
cultivar: 'bosc',
inSeason: false,
},
{
fruit: 'pear',
cultivar: 'bartlett',
inSeason: true,
},
],
}
type Fruits = typeof obj

test('getCsvHeaders returns the proper headers', () => {
const headers = getCsvHeadersFromJsonSchema(jsonSchema)
expect(headers).toStrictEqual([
'hash',
'filename',
'exifdata.Make',
'exifdata.Model',
'exifdata.DateTimeOriginal',
'exifdata.ModifyDate',
'exifdata.CreateDate',
'exifdata.GPSLatitudeRef',
'exifdata.GPSLatitude',
'exifdata.GPSLongitudeRef',
'exifdata.GPSLongitude',
'exifdata.GPSAltitudeRef',
'exifdata.GPSAltitude',
'exifdata.GPSTimeStamp',
'exifdata.GPSDateStamp',
'exifdata.ExifImageWidth',
'exifdata.ExifImageHeight',
'metadata.score',
'metadata.correction',
'metadata.remove',
'metadata.is_tf',
'metadata.id',
'metadata.label',
'metadata.area.x1',
'metadata.area.y1',
'metadata.area.x2',
'metadata.area.y2',
])
})
test('objToCsv correctly converts an object to a data string', () => {
const csv = objectToCsv(headers, (fruit: Fruits) =>
fruit.fruits.map((f) => [
fruit.id,
f.fruit,
f.cultivar,
f.inSeason,
]),
)(obj)
expect(csv).toEqual(
'id,fruit,cultivar,inSeason\n' +
'1,apple,gala,false\n' +
'1,apple,red delicious,true\n' +
'1,pear,bosc,false\n' +
'1,pear,bartlett,true',
)
})

test('objToCsv throws if row lengths are invalid', () => {
const csv = () =>
objectToCsv(headers, (fruit: Fruits) =>
fruit.fruits.map((f) => [f.fruit, f.cultivar, f.inSeason]),
)(obj)
expect(csv).toThrowError()
})
})

const jsonSchema: JSONSchema7 = {
$schema: 'http://json-schema.org/draft-07/schema#',
type: 'object',
properties: {
hash: { type: 'string' },
filename: { type: 'string' },
exifdata: {
type: 'object',
properties: {
Make: { type: 'string' },
Model: { type: 'string' },
DateTimeOriginal: { type: 'integer' },
ModifyDate: { type: 'integer' },
CreateDate: { type: 'integer' },
GPSLatitudeRef: { type: 'string' },
GPSLatitude: { type: 'number' },
GPSLongitudeRef: { type: 'string' },
GPSLongitude: { type: 'number' },
GPSAltitudeRef: { type: 'integer' },
GPSAltitude: { type: 'number' },
GPSTimeStamp: {
type: 'array',
items: { type: 'integer' },
},
GPSDateStamp: { type: 'string' },
ExifImageWidth: { type: 'integer' },
ExifImageHeight: { type: 'integer' },
},
required: [
'Make',
'Model',
'DateTimeOriginal',
'ModifyDate',
'CreateDate',
'GPSLatitudeRef',
'GPSLatitude',
'GPSLongitudeRef',
'GPSLongitude',
'GPSAltitudeRef',
'GPSAltitude',
'GPSTimeStamp',
'GPSDateStamp',
'ExifImageWidth',
'ExifImageHeight',
],
},
metadata: {
type: 'array',
items: {
type: 'object',
properties: {
score: { type: 'string' },
correction: { type: 'string' },
remove: { type: 'boolean' },
is_tf: { type: 'boolean' },
id: { type: 'string' },
label: { type: 'string' },
area: {
type: 'object',
properties: {
x1: { type: 'number' },
y1: { type: 'number' },
x2: { type: 'number' },
y2: { type: 'number' },
},
required: ['x1', 'y1', 'x2', 'y2'],
},
},
required: [
'score',
'correction',
'remove',
'is_tf',
'id',
'label',
'area',
],
},
},
},
required: ['hash', 'filename', 'exifdata', 'metadata'],
}
45 changes: 45 additions & 0 deletions frontend/src/lib/csv/schema-parser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { JSONSchema7Definition } from 'json-schema'
import { MapJsonProperties } from './types'

/**
* Traverses json schema objects and reduces them into a flat array using
* a passed in mapping function.
*
* Currently this function only finds objects and arrays, and assumes arrays all have a single type
*/
export const flatMapPropertiesRecursive =
<T>(fn: MapJsonProperties<T>, pathJoinStr: string = '_') =>
(props: Record<string, JSONSchema7Definition>, path?: string) =>
Object.entries(props).reduce((prev, [k, v]): T[] => {
const joinedKey = path ? `${path}${pathJoinStr}${k}` : k
const subProps = getObjOrArrProperties(v)
if (subProps) {
return prev.concat(
flatMapPropertiesRecursive(fn, pathJoinStr)(
subProps,
joinedKey,
),
)
}

return prev.concat(fn(joinedKey, props))
}, [] as T[])

/**
* Given a json schema definition, this function will return the list of object properties
* if the definition is an object or an array of objects
*/
export const getObjOrArrProperties = (d: JSONSchema7Definition) => {
if (typeof d === 'boolean') {
return undefined
}
if (d.properties) {
return d.properties
}
const item = Array.isArray(d.items) ? d.items[0] : d.items
if (typeof item !== 'boolean' && item?.properties) {
return item.properties
}

return undefined
}
6 changes: 6 additions & 0 deletions frontend/src/lib/csv/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { JSONSchema7Definition } from 'json-schema'

export type MapJsonProperties<T> = (
key: string,
props: Record<string, JSONSchema7Definition>,
) => T[]
Loading