Yolov5-Taco web model #7 (#30)

* Testing tf_web model export

* removing pyright

* Working version of taco dataset + initial build instructions

* Created links to code4sac and the github project

Added jupyter notebook files for new training as well as using existing
training data

* Adjusted new training jupyter notebook

.dockerignore

@@ -1 +1,3 @@
 **/node_modules
+./model
+./archive

.gitignore

@@ -13,3 +13,6 @@ data
 
 **/*.pyc
 .python-version
+**/*.zip
+pyrightconfig.json
+archive/

frontend/src/components/icon/code4sac.vue

@@ -0,0 +1,14 @@
+<template>
+    <v-layout align-center justify-center>
+        <svg class="icon" height="24" width="24" fill="currentColor">
+            <path
+                d="m 5.6757812,1.3300781 c -1.9141587,0 -3.0234374,1.0863015 -3.0234374,2.9667969 v 3.7832031 c 0,2.1070689 -0.5273327,2.8550879 -2.16406255,3.1269529 v 1.564453 c 1.63671245,0.271883 2.16406255,1.040606 2.16406255,3.125 v 3.785157 c 0,1.880495 1.1093368,2.966797 3.0234374,2.966797 H 6.9511719 V 21.175781 H 6.5351562 c -1.3593242,0 -1.7460937,-0.339437 -1.7460937,-1.517578 v -4.101562 c 0,-1.880496 -0.7218772,-2.989046 -2.2753906,-3.578125 1.6922366,-0.543765 2.2753906,-1.4742 2.2753906,-3.5585941 V 4.3203125 c 0,-1.1781585 0.3867695,-1.5175781 1.7460937,-1.5175781 H 6.9511719 V 1.3300781 Z m 11.3691408,0 v 1.4726563 h 0.443359 c 1.387058,0 1.775391,0.339437 1.775391,1.5175781 V 8.421875 c 0,1.857821 0.721877,2.989046 2.27539,3.578125 -1.553513,0.566416 -2.27539,1.678098 -2.27539,3.558594 v 4.099609 c 0,1.178159 -0.388333,1.517578 -1.775391,1.517578 h -0.443359 v 1.472657 h 1.304687 c 1.941892,0 3.050782,-1.086302 3.050782,-2.966797 v -3.783203 c 0,-2.084395 0.528111,-2.855089 2.109375,-3.126954 V 11.207031 C 21.928519,10.935149 21.400391,10.166425 21.400391,8.0820312 V 4.296875 c 0,-1.8804954 -1.10889,-2.9667969 -3.050782,-2.9667969 z M 8.3710938,3.6484375 V 12.451172 H 8.359375 v 0.859375 h 0.011719 v 7.019531 h 1.7128902 v -2.367187 h 3.822266 v 2.390625 h 1.744141 v -2.390625 h 0.0039 v -0.859375 h -0.0039 V 8.6132812 h 0.0039 v -0.859375 h -0.0039 V 5.3242188 H 13.90625 V 7.7539062 H 10.083984 V 3.6484375 Z m 2.1523442,4.9648437 h 2.945312 l -1.472656,1.4687498 z m -0.439454,0.4355469 1.472657,1.4707029 -1.472657,1.46875 z m 3.822266,0.00195 v 2.9355468 l -1.470703,-1.466797 z m -1.910156,1.9062498 1.498047,1.494141 h -2.996094 z m -1.91211,1.46875 0.02344,0.02539 h -0.02344 z m 3.822266,0.0039 v 0.02148 h -0.02148 z m -3.822266,0.880859 h 0.0098 l -0.0098,0.01172 z m 0.431641,0 h 2.960937 l -1.480468,1.476562 z m 3.382813,0 h 0.0078 v 0.0098 z m -3.814454,0.445312 1.472657,1.46875 -1.472657,1.470703 z m 3.822266,0.002 v 2.935547 l -1.470703,-1.46875 z m -1.910156,1.904297 1.445312,1.441407 h -2.890625 z"
+            />
+        </svg>
+    </v-layout>
+</template>
+<script>
+export default {
+    name: "code4sac",
+}
+</script>

frontend/src/components/model/canvas.vue

@@ -5,25 +5,27 @@
             <model-meta :item="item" :jtxt="json_txt" />
         </v-card-title>
         <v-card-text>
-            <canvas
-                :ref="item.key"
-                :width="resultWidth"
-                :height="resultHeight"
-            />
+            <canvas :ref="item.key" :width="maxwidth" :height="maxheight" />
         </v-card-text>
     </v-card>
 </template>
 <script>
+import * as tf from "@tensorflow/tfjs"
+
 export default {
     props: {
         item: {
             type: Object,
             required: true,
         },
-        model: {
+        name_map: {
             type: Object,
             required: true,
         },
+        model: {
+            type: tf.GraphModel,
+            required: true,
+        },
         pwidth: {
             type: String,
             required: true,
@@ -38,51 +40,137 @@ export default {
             dialog: false,
             maximize: false,
             json_txt: "",
+            maxwidth: 0,
+            maxheight: 0,
         }
     },
     async mounted() {
         console.log("loaded", this.item)
         const img = document.createElement("img")
-        const ref = await this.get_ref(this.item.key)
-
-        const ctx = ref.getContext("2d")
-
-        if (this.pwidth < this.item.width) {
-            this.resultWidth = this.pwidth
-            this.resultHeight =
-                (this.item.height * this.resultWidth) / this.item.width
+        if (this.item.width > this.pwidth) {
+            this.maxwidth = this.pwidth
+            this.maxheight =
+                (this.item.height * this.maxwidth) / this.item.width
         } else {
-            this.resultWidth = this.item.width
-            this.resultHeight = this.item.height
+            this.maxwidth = this.item.width
+            this.maxheight = this.item.height
         }
-
         img.onload = () => {
-            this.model.detect(img).then(async (predictions) => {
-                console.log("predictions", predictions)
-                this.json_txt = JSON.stringify(predictions, null, 2)
-                ctx.clearRect(0, 0, this.resultWidth, this.resultHeight)
-                ctx.drawImage(img, 0, 0, this.resultWidth, this.resultHeight)
-                predictions.forEach((prediction) => {
-                    const x = prediction.bbox[0]
-                    const y = prediction.bbox[1]
-                    const width = prediction.bbox[2]
-                    const height = prediction.bbox[3]
-                    ctx.strokeStyle = "red"
-                    ctx.lineWidth = 4
-                    ctx.strokeRect(x, y, width, height)
-                    ctx.font = "20px sans-serif"
-                    ctx.fillStyle = "red"
-                    ctx.fillText(
-                        `${prediction.class} ${prediction.score.toFixed(2)}`,
-                        x + width + 5,
-                        y + height + 10
-                    )
-                })
-            })
+            this.imageChange(img)
         }
-        img.width = this.resultWidth
-        img.height = this.resultHeight
+        img.width = this.item.width
+        img.height = this.item.height
         img.src = this.item.src
     },
+    methods: {
+        async imageChange(e) {
+            const c = await this.get_ref(this.item.key)
+            /**
+             * @type {HTMLCanvasElement}
+             */
+            const ctx = c.getContext("2d")
+            this.cropToCanvas(e, c, ctx)
+            let [modelWidth, modelHeight] = this.model.inputs[0].shape.slice(
+                1,
+                3
+            )
+            const input = tf.tidy(() => {
+                return tf.image
+                    .resizeBilinear(tf.browser.fromPixels(c), [
+                        modelWidth,
+                        modelHeight,
+                    ])
+                    .div(255.0)
+                    .expandDims(0)
+            })
+            this.model.executeAsync(input).then((res) => {
+                // Font options.
+                console.log("async exec", res)
+                const font = "14px sans-serif"
+                ctx.font = font
+                ctx.textBaseline = "top"
+
+                const [boxes, scores, classes, valid_detections] = res
+                const boxes_data = boxes.dataSync()
+                const scores_data = scores.dataSync()
+                const classes_data = classes.dataSync()
+                const valid_detections_data = valid_detections.dataSync()[0]
+                tf.dispose(res)
+                var i
+                let jarr = []
+                for (i = 0; i < valid_detections_data; ++i) {
+                    let [x1, y1, x2, y2] = boxes_data.slice(i * 4, (i + 1) * 4)
+                    x1 *= c.width
+                    x2 *= c.width
+                    y1 *= c.height
+                    y2 *= c.height
+                    const width = x2 - x1
+                    const height = y2 - y1
+                    const score = scores_data[i].toFixed(2)
+                    const name = this.name_map[classes_data[i]]
+                    jarr.push({
+                        x1: x1,
+                        y1: y1,
+                        x2: x2,
+                        y2: y2,
+                        width: width,
+                        height: height,
+                        score: score,
+                        name: name,
+                    })
+                    // Draw the bounding box.
+                    ctx.strokeStyle = "#00FFFF"
+                    ctx.lineWidth = 1
+                    ctx.strokeRect(x1, y1, width, height)
+
+                    // Draw the label background.
+                    ctx.fillStyle = "#00FFFF"
+                    const textWidth = ctx.measureText(`${name}:${score}`).width
+                    const textHeight = parseInt(font, 10) // base 10
+                    ctx.fillRect(x1, y1, textWidth + 4, textHeight + 4)
+                }
+                this.json_txt = JSON.stringify(jarr, null, 2)
+                for (i = 0; i < valid_detections_data; ++i) {
+                    let [x1, y1, ,] = boxes_data.slice(i * 4, (i + 1) * 4)
+                    x1 *= c.width
+                    y1 *= c.height
+                    const score = scores_data[i].toFixed(2)
+                    const name = this.name_map[classes_data[i]]
+
+                    // Draw the text last to ensure it's on top.
+                    ctx.fillStyle = "#000000"
+                    ctx.fillText(`${name}:${score}`, x1, y1)
+                }
+                ctx.width = c.width
+                ctx.height = c.height
+            })
+        },
+        cropToCanvas(image, canvas, ctx) {
+            let naturalWidth = image.naturalWidth
+            let naturalHeight = image.naturalHeight
+            ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height)
+            ctx.fillStyle = "#000000"
+            ctx.fillRect(0, 0, canvas.width, canvas.height)
+            const ratio = Math.min(
+                canvas.width / naturalWidth,
+                canvas.height / naturalHeight
+            )
+            const newWidth = Math.round(naturalWidth * ratio)
+            const newHeight = Math.round(naturalHeight * ratio)
+            console.log(newWidth, newHeight)
+            console.log(canvas.width, canvas.height)
+            ctx.drawImage(
+                image,
+                0,
+                0,
+                naturalWidth,
+                naturalHeight,
+                (canvas.width - newWidth) / 2,
+                (canvas.height - newHeight) / 2,
+                newWidth,
+                newHeight
+            )
+        },
+    },
 }
 </script>