-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CSV summary Addition to production (#178)
* Adding a link to Running-man-01/trashai_nbs project * Bump vm2 from 3.9.17 to 3.9.19 in /infra (#162) Bumps [vm2](https://github.com/patriksimek/vm2) from 3.9.17 to 3.9.19. - [Release notes](https://github.com/patriksimek/vm2/releases) - [Changelog](https://github.com/patriksimek/vm2/blob/master/CHANGELOG.md) - [Commits](patriksimek/vm2@3.9.17...3.9.19) --- updated-dependencies: - dependency-name: vm2 dependency-type: indirect ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump vite from 3.2.5 to 3.2.7 in /infra (#165) Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 3.2.5 to 3.2.7. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v3.2.7/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v3.2.7/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump vite from 3.0.4 to 3.2.7 in /frontend (#164) Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 3.0.4 to 3.2.7. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v3.2.7/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v3.2.7/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fixing build and fixing local (#170) * Add testing instructions and auto tests (#171) * Adding a link to Running-man-01/trashai_nbs project (#168) * update frontend app doc * Add manual instructions. * Added test-ids * Add test-ids to the navigation tabs * Add smoke test draft * Add placeholder file * Add photos for manual testing instructions * Upgrade node version * Remove cypress * Add test ids to summary page * Jest + Puppeteer setup * Add smoke test * Added back manual instructions * Add images for manual instructions * Add instruction for running auto tests --------- Co-authored-by: Dan Fey <[email protected]> * Adding schema files to download (#173) * adding schema files to the zip download * revert yarn change * fixing list * indentation * 124 summary csv files (#175) * aAdding summary csv reports * adding comments * fixing off by 1 error in summary counts (#177) --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Dan Fey <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Zesky665 <[email protected]>
- Loading branch information
4 people
authored
Jun 27, 2023
1 parent
1daab77
commit 9b77297
Showing
12 changed files
with
4,617 additions
and
2,172 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,3 +23,4 @@ pnpm-debug.log* | |
| *.sln | ||
| *.sw? | ||
| .log | ||
| package-lock.json | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| module.exports = { | ||
| globalSetup: './setup.js', | ||
| globalTeardown: './teardown.js', | ||
| testEnvironment: './puppeteer_environment.js', | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| /** @type {import('ts-jest').JestConfigWithTsJest} */ | ||
| module.exports = { | ||
| globalSetup: './setup.js', | ||
| globalTeardown: './teardown.js', | ||
| testEnvironment: './puppeteer_environment.js', | ||
| } | ||
| preset: 'ts-jest', | ||
| testEnvironment: 'node', | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| import { JSONSchema7Definition } from 'json-schema' | ||
| import { | ||
| flatMapPropertiesRecursive, | ||
| getObjOrArrProperties, | ||
| } from './schema-parser' | ||
| import { MapJsonProperties } from './types' | ||
|
|
||
| export const objectToCsv = | ||
| <T>( | ||
| headers: string[], | ||
| fnFlatten: (obj: T) => (string | number | boolean)[][], | ||
| joinStr: string = ',', | ||
| ) => | ||
| (obj: T) => { | ||
| const csvData = fnFlatten(obj) | ||
| const errorRows = csvData.filter((v) => v.length !== headers.length) | ||
| if (errorRows.length !== 0) { | ||
| throw new Error( | ||
| `All csv rows must have the same length as the headers array ${JSON.stringify( | ||
| headers, | ||
| )}, see incorrect rows: ${JSON.stringify(errorRows)}`, | ||
| ) | ||
| } | ||
| return [headers, ...csvData].map((row) => row.join(joinStr)).join('\n') | ||
| } | ||
|
|
||
| /** | ||
| * Traverses a json schema object and returns a flat array | ||
| * representing the keys of all objects and subobjects. | ||
| * Maintains nesting representation by joining keys together with '.' | ||
| */ | ||
| export const getCsvHeadersFromJsonSchema = ( | ||
| jsonSchema: JSONSchema7Definition, | ||
| ) => { | ||
| const props = getObjOrArrProperties(jsonSchema) | ||
| return props ? flatMapPropertiesRecursive(getJsonKeys, '.')(props) : [] | ||
| } | ||
|
|
||
| const getJsonKeys: MapJsonProperties<string> = (k) => [k] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| import { describe, expect, test } from '@jest/globals' | ||
| import { JSONSchema7 } from 'json-schema' | ||
| import { getCsvHeadersFromJsonSchema, objectToCsv } from './csv' | ||
|
|
||
| describe('csv', () => { | ||
| const headers = ['id', 'fruit', 'cultivar', 'inSeason'] | ||
| const obj = { | ||
| id: 1, | ||
| fruits: [ | ||
| { | ||
| fruit: 'apple', | ||
| cultivar: 'gala', | ||
| inSeason: false, | ||
| }, | ||
| { | ||
| fruit: 'apple', | ||
| cultivar: 'red delicious', | ||
| inSeason: true, | ||
| }, | ||
| { | ||
| fruit: 'pear', | ||
| cultivar: 'bosc', | ||
| inSeason: false, | ||
| }, | ||
| { | ||
| fruit: 'pear', | ||
| cultivar: 'bartlett', | ||
| inSeason: true, | ||
| }, | ||
| ], | ||
| } | ||
| type Fruits = typeof obj | ||
|
|
||
| test('getCsvHeaders returns the proper headers', () => { | ||
| const headers = getCsvHeadersFromJsonSchema(jsonSchema) | ||
| expect(headers).toStrictEqual([ | ||
| 'hash', | ||
| 'filename', | ||
| 'exifdata.Make', | ||
| 'exifdata.Model', | ||
| 'exifdata.DateTimeOriginal', | ||
| 'exifdata.ModifyDate', | ||
| 'exifdata.CreateDate', | ||
| 'exifdata.GPSLatitudeRef', | ||
| 'exifdata.GPSLatitude', | ||
| 'exifdata.GPSLongitudeRef', | ||
| 'exifdata.GPSLongitude', | ||
| 'exifdata.GPSAltitudeRef', | ||
| 'exifdata.GPSAltitude', | ||
| 'exifdata.GPSTimeStamp', | ||
| 'exifdata.GPSDateStamp', | ||
| 'exifdata.ExifImageWidth', | ||
| 'exifdata.ExifImageHeight', | ||
| 'metadata.score', | ||
| 'metadata.correction', | ||
| 'metadata.remove', | ||
| 'metadata.is_tf', | ||
| 'metadata.id', | ||
| 'metadata.label', | ||
| 'metadata.area.x1', | ||
| 'metadata.area.y1', | ||
| 'metadata.area.x2', | ||
| 'metadata.area.y2', | ||
| ]) | ||
| }) | ||
| test('objToCsv correctly converts an object to a data string', () => { | ||
| const csv = objectToCsv(headers, (fruit: Fruits) => | ||
| fruit.fruits.map((f) => [ | ||
| fruit.id, | ||
| f.fruit, | ||
| f.cultivar, | ||
| f.inSeason, | ||
| ]), | ||
| )(obj) | ||
| expect(csv).toEqual( | ||
| 'id,fruit,cultivar,inSeason\n' + | ||
| '1,apple,gala,false\n' + | ||
| '1,apple,red delicious,true\n' + | ||
| '1,pear,bosc,false\n' + | ||
| '1,pear,bartlett,true', | ||
| ) | ||
| }) | ||
|
|
||
| test('objToCsv throws if row lengths are invalid', () => { | ||
| const csv = () => | ||
| objectToCsv(headers, (fruit: Fruits) => | ||
| fruit.fruits.map((f) => [f.fruit, f.cultivar, f.inSeason]), | ||
| )(obj) | ||
| expect(csv).toThrowError() | ||
| }) | ||
| }) | ||
|
|
||
| const jsonSchema: JSONSchema7 = { | ||
| $schema: 'http://json-schema.org/draft-07/schema#', | ||
| type: 'object', | ||
| properties: { | ||
| hash: { type: 'string' }, | ||
| filename: { type: 'string' }, | ||
| exifdata: { | ||
| type: 'object', | ||
| properties: { | ||
| Make: { type: 'string' }, | ||
| Model: { type: 'string' }, | ||
| DateTimeOriginal: { type: 'integer' }, | ||
| ModifyDate: { type: 'integer' }, | ||
| CreateDate: { type: 'integer' }, | ||
| GPSLatitudeRef: { type: 'string' }, | ||
| GPSLatitude: { type: 'number' }, | ||
| GPSLongitudeRef: { type: 'string' }, | ||
| GPSLongitude: { type: 'number' }, | ||
| GPSAltitudeRef: { type: 'integer' }, | ||
| GPSAltitude: { type: 'number' }, | ||
| GPSTimeStamp: { | ||
| type: 'array', | ||
| items: { type: 'integer' }, | ||
| }, | ||
| GPSDateStamp: { type: 'string' }, | ||
| ExifImageWidth: { type: 'integer' }, | ||
| ExifImageHeight: { type: 'integer' }, | ||
| }, | ||
| required: [ | ||
| 'Make', | ||
| 'Model', | ||
| 'DateTimeOriginal', | ||
| 'ModifyDate', | ||
| 'CreateDate', | ||
| 'GPSLatitudeRef', | ||
| 'GPSLatitude', | ||
| 'GPSLongitudeRef', | ||
| 'GPSLongitude', | ||
| 'GPSAltitudeRef', | ||
| 'GPSAltitude', | ||
| 'GPSTimeStamp', | ||
| 'GPSDateStamp', | ||
| 'ExifImageWidth', | ||
| 'ExifImageHeight', | ||
| ], | ||
| }, | ||
| metadata: { | ||
| type: 'array', | ||
| items: { | ||
| type: 'object', | ||
| properties: { | ||
| score: { type: 'string' }, | ||
| correction: { type: 'string' }, | ||
| remove: { type: 'boolean' }, | ||
| is_tf: { type: 'boolean' }, | ||
| id: { type: 'string' }, | ||
| label: { type: 'string' }, | ||
| area: { | ||
| type: 'object', | ||
| properties: { | ||
| x1: { type: 'number' }, | ||
| y1: { type: 'number' }, | ||
| x2: { type: 'number' }, | ||
| y2: { type: 'number' }, | ||
| }, | ||
| required: ['x1', 'y1', 'x2', 'y2'], | ||
| }, | ||
| }, | ||
| required: [ | ||
| 'score', | ||
| 'correction', | ||
| 'remove', | ||
| 'is_tf', | ||
| 'id', | ||
| 'label', | ||
| 'area', | ||
| ], | ||
| }, | ||
| }, | ||
| }, | ||
| required: ['hash', 'filename', 'exifdata', 'metadata'], | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| import { JSONSchema7Definition } from 'json-schema' | ||
| import { MapJsonProperties } from './types' | ||
|
|
||
| /** | ||
| * Traverses json schema objects and reduces them into a flat array using | ||
| * a passed in mapping function. | ||
| * | ||
| * Currently this function only finds objects and arrays, and assumes arrays all have a single type | ||
| */ | ||
| export const flatMapPropertiesRecursive = | ||
| <T>(fn: MapJsonProperties<T>, pathJoinStr: string = '_') => | ||
| (props: Record<string, JSONSchema7Definition>, path?: string) => | ||
| Object.entries(props).reduce((prev, [k, v]): T[] => { | ||
| const joinedKey = path ? `${path}${pathJoinStr}${k}` : k | ||
| const subProps = getObjOrArrProperties(v) | ||
| if (subProps) { | ||
| return prev.concat( | ||
| flatMapPropertiesRecursive(fn, pathJoinStr)( | ||
| subProps, | ||
| joinedKey, | ||
| ), | ||
| ) | ||
| } | ||
|
|
||
| return prev.concat(fn(joinedKey, props)) | ||
| }, [] as T[]) | ||
|
|
||
| /** | ||
| * Given a json schema definition, this function will return the list of object properties | ||
| * if the definition is an object or an array of objects | ||
| */ | ||
| export const getObjOrArrProperties = (d: JSONSchema7Definition) => { | ||
| if (typeof d === 'boolean') { | ||
| return undefined | ||
| } | ||
| if (d.properties) { | ||
| return d.properties | ||
| } | ||
| const item = Array.isArray(d.items) ? d.items[0] : d.items | ||
| if (typeof item !== 'boolean' && item?.properties) { | ||
| return item.properties | ||
| } | ||
|
|
||
| return undefined | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| import { JSONSchema7Definition } from 'json-schema' | ||
|
|
||
| export type MapJsonProperties<T> = ( | ||
| key: string, | ||
| props: Record<string, JSONSchema7Definition>, | ||
| ) => T[] |
Oops, something went wrong.