Merge pull request #109 from JetBrains-Research/master-dev

Release 0.6

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2
+version: 2.1
 jobs:
   # build with machine executor
   build:
@@ -15,91 +15,52 @@ jobs:
     machine:
       enabled: true
     working_directory: ~/astminer
+    environment:
+      GIT_BRANCH: << pipeline.git.branch >>
     steps:
       - checkout:
           path: ~/astminer
-      - run: ./gradlew bintrayUpload
-
-  build-cli:
-    machine:
-      enabled: true
-    working_directory: ~/astminer-cli
-    steps:
-      - checkout:
-          path: ~/astminer-cli
-      - run: cd astminer-cli && ./gradlew shadowJar
+      - run: ./gradlew bintrayUpload "-PbranchName=$GIT_BRANCH"
 
-  release-cli:
+  release-dev:
     machine:
       enabled: true
-    working_directory: ~/astminer-cli
-    steps:
-      - checkout:
-          path: ~/astminer-cli
-      - run: cd astminer-cli && ./gradlew bintrayUpload
-
-  # run the Python example
-  py_example:
-    docker:
-      - image: openkbs/jdk-mvn-py3
-    working_directory: ~/astminer/py_example
-    # Unless we restrict JVM memory, Kotlin compilation fails
+    working_directory: ~/astminer
     environment:
-      JVM_OPTS: -Xmx2048m
-      GRADLE_OPTS: -Xmx1536m -XX:+HeapDumpOnOutOfMemoryError -Dorg.gradle.parallel=false -Dkotlin.incremental=false
+      CI_VERSION: << pipeline.number >>
+      GIT_BRANCH: << pipeline.git.branch >>
     steps:
       - checkout:
           path: ~/astminer
-      - run:
-          name: Disable gradle daemon
-          command: mkdir -p ~/.gradle && echo "org.gradle.daemon=false" >> ~/.gradle/gradle.properties
-      - run:
-          name: Create environment
-          command: python3 -m virtualenv env
-      - run:
-          name: Activate environment
-          command: source env/bin/activate
-      - run:
-          name: Install requirements
-          command: pip3 install --user -r requirements.txt
-      - run:
-          name: Run processing
-          command: python3 run_example.py
+      - run: ./gradlew bintrayUpload "-PciVersion=1.$CI_VERSION" "-PbranchName=$GIT_BRANCH"
 
 
 workflows:
   version: 2
-  build:
-    jobs:
-      - build
 
   # release with manual approval in CircleCI app
-  release:
+  deploy-library:
     jobs:
       - build
       - approve-release:
           type: approval
           requires:
             - build
+          filters:
+            branches:
+              only:
+                - master
       - release:
           requires:
             - approve-release
-
-  build-cli:
-    jobs:
-      - build-cli
-
-  release-cli:
-    jobs:
-      - build-cli
-      - approve-release-cli:
-          type: approval
-          requires:
-            - build-cli
-      - release-cli:
+          filters:
+            branches:
+              only:
+                - master
+      - release-dev:
           requires:
-            - approve-release-cli
-
-#  py_example:
-#    jobs:
-#      - py_example
+            - build
+          filters:
+            branches:
+              only:
+                - master-dev

README.md

@@ -14,34 +14,10 @@ Supported languages of the input:
 
 ### Version history
 
-#### 0.5
-
-* Beta of Javascript support
-* Storage of ASTs in [DOT format](https://www.graphviz.org/doc/info/lang.html)
-* Minor fixes
-
-#### 0.4
-
-* Support of code2vec output format
-* Extraction of ASTs and path-based representations of individual methods
-* Extraction of data for the task of method name prediction ([code2vec paper](https://arxiv.org/abs/1803.09473))
-
-#### 0.3
-
-* Support of C/C++ via [FuzzyC2CPG parser](https://github.com/ShiftLeftSecurity/fuzzyc2cpg)
-
-#### 0.2
-
-* Mining of ASTs
-
-#### 0.1
-* astminer is available via Maven Central
-* Support of Java and Python
-* Mining of [path-based representations of code](https://arxiv.org/pdf/1803.09544.pdf)
-
+See [changelog](changelog.md)
 
 ## About
-Astminer is an offspring of an internal utility from our ongoing research project.
+Astminer was first implemented as a part of pipeline in the [the code style extraction project](https://arxiv.org/abs/2002.03997) and later converted into a reusable tool.
 
 Currently it supports extraction of:
 * Path-based representations of files
@@ -56,7 +32,37 @@ For the output format, see the section below.
 
 ### Use as CLI
 
-See a [subfolder](/astminer-cli) containing CLI and its description. It can be extended if needed.
+1. Run  `./gradlew shadowJar` in project directory 
+2. Now you can use shell script to run cli  `./cli.sh optionName parameters`, where `optionName` is one of the following options:
+
+#### Preprocess
+
+Run preprocessing on C/C++ project to unfold `#define` directives. 
+In other tasks, if you feed C/C++ file with macroses, they will be dropped as well as their appearances in code. 
+```shell script
+./cli.sh preprocess --project path/to/project --output path/to/preprocessedProject
+```
+#### Parse
+
+Extract ASTs from all the files in supported languages.
+```shell script
+./cli.sh parse --lang py,java,c,cpp --project path/to/project --output path/to/result --storage dot
+```
+
+#### PathContexts
+
+Extract path contexts from all the files in supported languages and store in form `fileName triplesOfPathContexts`.
+```shell script
+./cli.sh pathContexts --lang py,java,c,cpp --project path/to/project --output path/to/results --maxL L --maxW W --maxContexts C --maxTokens T --maxPaths P
+```
+
+#### Code2vec
+
+Extract data suitable as input for [code2vec](https://github.com/tech-srl/code2vec) model.
+Parse all files written in specified language into ASTs, split into methods, and store in form `method|name triplesOfPathContexts`.
+```shell script
+./cli.sh code2vec --lang py,java,c,cpp --project path/to/project --output path/to/results --maxL L --maxW W --maxContexts C --maxTokens T --maxPaths P  --split-tokens --granularity method
+```
 
 ### Integrate in your mining pipeline
 
@@ -71,7 +77,7 @@ repositories {
 }
 
 dependencies {
-    compile 'io.github.vovak.astminer:astminer:0.5'
+    compile 'io.github.vovak.astminer:astminer:0.6'
 }
 ```
 
@@ -82,7 +88,7 @@ repositories {
 }
 
 dependencies {
-    compile("io.github.vovak.astminer", "astminer", "0.5")
+    compile("io.github.vovak.astminer", "astminer", "0.6")
 }
 ```