Merge pull request #36 from TailUFPB/develop

Mudar a forma de criar o executavel para que seja mais rapido

.github/workflows/develop.yml

@@ -1,69 +1,30 @@
 name: CI
-# Controls when the workflow will run
 on:
   push:
     branches: ["develop"]
 
-# Prevent duplicate workflows from running
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  # Static tests don't involve any logic or context.
-  # They are just a set of tests that can detect if we are not introducing any faulty code.
-  static-test:
-    name: 🔬 Static tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-
-      - name: ⎔ Setup Node
-        uses: actions/setup-node@v3
-        with:
-          cache: "npm"
-
-      - name: 🟨 Setup Python
-        uses: actions/setup-python@v3
-
-      - name: 📦 Install Node dependencies
-        run: npm install
-
-  # # Unit tests are tests that are not dependent on any external service.
-  # # Usually, they are tests that are testing the logic of a specific function or component.
-  unit-test:
-    needs: [static-test]
-    name: 🚦 Unit tests
-    runs-on: ubuntu-latest
+  build:
+    name: 🏗️ Build
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
     steps:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 2
 
-      - name: ⎔ Setup Node
-        uses: actions/setup-node@v3
+      - name: Set up conda
+        uses: conda-incubator/setup-miniconda@v2
         with:
-          cache: "npm"
-
-      - name: 🟨 Setup Python
-        uses: actions/setup-python@v3
-
-      - name: 📦 Install dependencies
-        run: npm install
+          activate-environment: default
+          environment-file: api/environment.yml
 
-      - name: 🚦 Run unit tests
-        run: npm test
-
-  # # Integration tests are tests that are dependent on external services.
-  integration-test:
-    needs: [static-test]
-    name: 🚥 Integration tests
-    runs-on: ubuntu-latest
-    steps:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v4
         with:
@@ -76,65 +37,26 @@ jobs:
 
       - name: 🟨 Setup Python
         uses: actions/setup-python@v3
-
-      - name: 📦 Install dependencies
-        run: npm install
-
-      # - name: 🐳 Docker compose
-      #   run:
-      #     docker-compose up -d && sleep 3 && pnpm prisma migrate reset --force
-      #     --skip-seed
-
-      - name: 🚦 Run integration tests
-        run: npm test
-
-  # Create Build
-  build:
-    needs: [static-test, unit-test, integration-test]
-    name: 🏗️ Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Wine
-        uses: actions/setup-node@v2
-        with:
-          node-version: "14"
-
-      - name: Install Wine64
-        run: sudo apt update && sudo apt install wine64
-
-      - name: Install Wine32
-        run: |
-          sudo dpkg --add-architecture i386
-          sudo apt-get update
-          sudo apt-get install wine32
-
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v4
         with:
-          fetch-depth: 2
+          python-version: 3.12.3
 
-      - name: ⎔ Setup Node
-        uses: actions/setup-node@v3
-        with:
-          cache: "npm"
+      - name: Install pip dependencies
+        run: pip install -r api/requirements.txt pyinstaller==6.6.0
 
-      - name: 🟨 Setup Python
-        uses: actions/setup-python@v3
+      - name: Build flask exe
+        run: pyinstaller api/app.spec
 
-      - name: 📦 Install dependencies
+      - name: Install dependencies
         run: npm install
 
-      - name: 🏗️ Build
-        run: npm run build
-
-      #- name: 📦 Electron Package
-      #  run: npx electron-packager . LinguifAI --platform=win32 --arch=x64 --out=dist
-
       - name: 📦 Electron Builder
         run: npm run electron:package:win
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Print directory tree
+        run: tree
+
       - name: Get latest release number
         id: get_latest_release
         uses: actions/github-script@v4
@@ -154,7 +76,7 @@ jobs:
             console.log(`::set-output name=new_version::${newVersion}`);
 
       - name: Rename file
-        run: mv -f dist/LinguifAI\ Setup\ 0.1.0.exe dist/LinguifAI\ Setup\ ${{ steps.get_latest_release.outputs.new_version }}.exe
+        run: ren "dist\LinguifAI Setup 0.1.0.exe" "LinguifAI Setup ${{ steps.get_latest_release.outputs.new_version }}.exe"
 
       - name: Create Release
         id: create_release
@@ -165,5 +87,4 @@ jobs:
           tag_name: v${{ steps.get_latest_release.outputs.new_version }}
           name: v${{ steps.get_latest_release.outputs.new_version }}
           prerelease: true
-          body: |
-            Descrição do release aqui
+          body: ${{ github.event.head_commit.message }}

.github/workflows/main.yml

@@ -92,21 +92,20 @@ jobs:
   build:
     needs: [static-test, unit-test, integration-test]
     name: 🏗️ Build
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
     steps:
-      - name: Set up Wine
-        uses: actions/setup-node@v2
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
         with:
-          node-version: "14"
-
-      - name: Install Wine64
-        run: sudo apt update && sudo apt install wine64
+          fetch-depth: 2
 
-      - name: Install Wine32
-        run: |
-          sudo dpkg --add-architecture i386
-          sudo apt-get update
-          sudo apt-get install wine32
+      - name: Set up conda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: default
+          environment-file: api/environment.yml
 
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v4
@@ -120,21 +119,26 @@ jobs:
 
       - name: 🟨 Setup Python
         uses: actions/setup-python@v3
+        with:
+          python-version: 3.12.3
 
-      - name: 📦 Install dependencies
-        run: npm install
+      - name: Install pip dependencies
+        run: pip install -r api/requirements.txt pyinstaller==6.6.0
 
-      - name: 🏗️ Build
-        run: npm run build
+      - name: Build flask exe
+        run: pyinstaller api/app.spec
 
-      #- name: 📦 Electron Package
-      #  run: npx electron-packager . LinguifAI --platform=win32 --arch=x64 --out=dist
+      - name: Install dependencies
+        run: npm install
 
       - name: 📦 Electron Builder
         run: npm run electron:package:win
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Print directory tree
+        run: tree
+
       - name: Get latest release number
         id: get_latest_release
         uses: actions/github-script@v4
@@ -145,7 +149,7 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               per_page: 10
-            });    
+            });
 
             const latestPreRelease = response.data[0];
             const preReleaseTag = latestPreRelease.name;
@@ -154,8 +158,21 @@ jobs:
             console.log(`::set-output name=new_version::${newVersion}`);
 
       - name: Rename file
-        run: mv -f dist/LinguifAI\ Setup\ 0.1.0.exe dist/LinguifAI\ Setup\ ${{ steps.get_latest_release.outputs.new_version }}.exe
+        run: ren "dist\LinguifAI Setup 0.1.0.exe" "LinguifAI Setup ${{ steps.get_latest_release.outputs.new_version }}.exe"
 
+      - name: Get latest commit message
+        id: get_latest_commit_message
+        uses: actions/github-script@v4
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const response = await github.repos.listCommits({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              per_page: 1,
+            });
+            console.log(`::set-output name=commit_message::${response.data[0].commit.message}`);
+      
       - name: Create Release
         id: create_release
         uses: softprops/action-gh-release@v1
@@ -165,5 +182,4 @@ jobs:
           tag_name: v${{ steps.get_latest_release.outputs.new_version }}
           name: v${{ steps.get_latest_release.outputs.new_version }}
           prerelease: true
-          body: |
-            Descrição do release aqui
+          body: ${{ steps.get_latest_commit_message.outputs.commit_message }}

api/DataProcesser.py

@@ -2,13 +2,18 @@
 from NbNewsModel import news_prediction
 from NbEmotionsModel import make_prediction
 from available_classifiers import get_available_classifiers
+from sklearn.pipeline import make_pipeline
+
+# bag of words
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+from sklearn.model_selection import train_test_split
+from sklearn.naive_bayes import MultinomialNB
 
 import Neural_Network2
-import pickle
 import re
 import joblib
 import numpy as np
-import string
 import os
 import pandas as pd
 import torch
@@ -30,7 +35,8 @@ def handle_classify(self, df, classifier):
         classifier_switcher = get_available_classifiers() # id: nome_arquivo
         model_name = classifier_switcher[classifier]
         if model_name.endswith('.pkl'):
-            return self.pretrained_predict(df, model_name)
+            pipeline = self.get_pipeline(model_name)
+            return self.pretrained_predict(df, pipeline)
         else:
             return self.trained_predict(df, model_name)
         #classifier_switcher = {
@@ -41,6 +47,20 @@ def handle_classify(self, df, classifier):
 
         #return classifier_switcher.get(classifier, lambda: "Invalid Classifier")(df)
 
+    def get_pipeline(self, model_name):
+        if model_name=="emotion_pipeline.pkl":
+            df = pd.read_csv('api/training_df/tweet_emotions.csv')
+            train_data, test_data, train_target, test_target = train_test_split(df['content'], df['sentiment'], test_size=0.2, shuffle=True)
+        elif model_name=="hate_speech.pkl":
+            df = pd.read_csv('api/training_df/nb_hatespeech.csv', sep=';')
+            train_data, test_data, train_target, test_target = train_test_split(df['comment'], df['isHate'], test_size=0.2, shuffle=True)
+        elif model_name=="text_classification_pipeline.pkl":
+            df = pd.read_csv('api/training_df/nb_news.csv')
+            train_data, test_data, train_target, test_target = train_test_split(df['short_description'], df['category'], test_size=0.2, shuffle=True)
+        else:
+            return None
+        return make_pipeline(TfidfVectorizer(), MultinomialNB()).fit(train_data, train_target)
+
     def generate_statistics(self, df):
         unique_labels = df['output_column'].unique()
 
@@ -67,19 +87,15 @@ def classify_emotions(self, df):
         df['output_column'] = df['input_column'].apply(make_prediction)
         return df
 
-    def lin_regression_model(self, df):
-        df['output_column'] = df['input_column'].apply(make_prediction_nblin)
-        return df
+    # def lin_regression_model(self, df):
+    #     df['output_column'] = df['input_column'].apply(make_prediction_nblin)
+    #     return df
 
     def nb_news_application(self, df):
         df['output_column'] = df['input_column'].apply(news_prediction)
         return df
 
-    def pretrained_predict(self, df, model_name):
-        model_file = f'api/models/{model_name}'
-        with open(model_file, 'rb') as model:
-            pipeline = pickle.load(model)
-
+    def pretrained_predict(self, df, pipeline):
         texts_to_predict = df['input_column']
         texts_to_predict = [str(text) for text in texts_to_predict]
         predictions = pipeline.predict(texts_to_predict)