Dev to GraphQL-API-Experiment

.gitignore

@@ -4,13 +4,12 @@ env.txt
 docker_env.txt
 pyenv.txt
 augur_export_env.sh
-.DS_Store
+*DS_Store
 *.config.json
 !docker.config.json
 config.yml
 reports.yml
 
-
 node_modules/
 .idea/
 logs/

README.md

@@ -1,4 +1,4 @@
-# Augur NEW Release v0.52.0
+# Augur NEW Release v0.53.1
 
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 
@@ -8,7 +8,7 @@
 ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.52.0
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.53.1
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard

augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py

@@ -26,11 +26,34 @@ def contributor_breadth_model() -> None:
     tool_version = '0.0.1'
     data_source = 'GitHub API'
 
-
+    # This version of the query pulls contributors who have not had any data collected yet
+    # To the top of the list
     cntrb_login_query = s.sql.text("""
-        SELECT DISTINCT gh_login, cntrb_id 
-        FROM augur_data.contributors 
-        WHERE gh_login IS NOT NULL
+            SELECT DISTINCT
+                gh_login,
+                cntrb_id 
+            FROM
+                (
+                SELECT DISTINCT
+                    gh_login,
+                    cntrb_id,
+                    data_collection_date 
+                FROM
+                    (
+                    SELECT DISTINCT
+                        contributors.gh_login,
+                        contributors.cntrb_id,
+                        contributor_repo.data_collection_date :: DATE 
+                    FROM
+                        contributor_repo
+                        RIGHT OUTER JOIN contributors ON contributors.cntrb_id = contributor_repo.cntrb_id 
+                        AND contributors.gh_login IS NOT NULL 
+                    ORDER BY
+                        contributor_repo.data_collection_date :: DATE NULLS FIRST 
+                    ) A 
+                ORDER BY
+                data_collection_date DESC NULLS FIRST 
+                ) b
     """)
 
     result = engine.execute(cntrb_login_query)

augur/tasks/db/refresh_materialized_views.py

@@ -6,27 +6,128 @@
 
 from augur.tasks.init.celery_app import celery_app as celery
 from augur.application.db.session import DatabaseSession
+from augur.application.logs import AugurLogger
 
 
 @celery.task
 def refresh_materialized_views():
 
+    #self.logger = AugurLogger("data_collection_jobs").get_logger()
+
     from augur.tasks.init.celery_app import engine
 
     logger = logging.getLogger(refresh_materialized_views.__name__)
+    #self.logger = logging.getLogger(refresh_materialized_views.__name__)
 
-    refresh_view_query = s.sql.text("""    
+    mv1_refresh = s.sql.text("""    
                 REFRESH MATERIALIZED VIEW concurrently augur_data.api_get_all_repo_prs with data;
+                COMMIT; 
+    """)
+
+    mv2_refresh = s.sql.text("""    
                 REFRESH MATERIALIZED VIEW concurrently augur_data.api_get_all_repos_commits with data;
+                COMMIT; 
+    """)
+
+    mv3_refresh = s.sql.text("""    
                 REFRESH MATERIALIZED VIEW concurrently augur_data.api_get_all_repos_issues with data;
+                COMMIT; 
+    """)
+
+    mv4_refresh = s.sql.text("""    
                 REFRESH MATERIALIZED VIEW concurrently augur_data.augur_new_contributors with data;
+                COMMIT; 
+    """)
+    mv5_refresh = s.sql.text("""    
                 REFRESH MATERIALIZED VIEW concurrently augur_data.explorer_commits_and_committers_daily_count with data;
+                COMMIT; 
+    """)
+
+    mv6_refresh = s.sql.text("""    
                 REFRESH MATERIALIZED VIEW concurrently augur_data.explorer_new_contributors with data;
+                COMMIT; 
+    """)
+
+    mv7_refresh = s.sql.text("""    
                 REFRESH MATERIALIZED VIEW concurrently augur_data.explorer_entry_list with data;
-                REFRESH MATERIALIZED VIEW concurrently augur_data.explorer_new_contributors with data;
+                COMMIT; 
+    """)
+
+    mv_8_refresh = s.sql.text("""    
+
                 REFRESH MATERIALIZED VIEW concurrently augur_data.explorer_contributor_actions with data;
+                COMMIT; 
     """)
 
-    with DatabaseSession(logger, engine) as session:
 
-        session.execute_sql(refresh_view_query)
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv1_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv1_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv2_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv3_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv4_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv5_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv6_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv7_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+    try: 
+        with DatabaseSession(logger, engine) as session:
+            session.execute_sql(mv8_refresh)
+    except Exception as e: 
+        logger.info(f"error is {e}")
+        pass 
+
+
+
+
+
+
+

augur/tasks/github/pull_requests/tasks.py

@@ -333,7 +333,7 @@ def collect_pull_request_reviews(repo_git: str) -> None:
 
         pr_count = len(prs)
 
-        all_raw_pr_reviews = []
+        all_pr_reviews = {}
         for index, pr in enumerate(prs):
 
             pr_number = pr.pr_src_number
@@ -343,40 +343,46 @@ def collect_pull_request_reviews(repo_git: str) -> None:
 
             pr_review_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/reviews"
 
-            pr_reviews = GithubPaginator(pr_review_url, manifest.key_auth, logger)
-
-            for page_data, page in pr_reviews.iter_pages():
+            pr_reviews = []
+            pr_reviews_generator = GithubPaginator(pr_review_url, manifest.key_auth, logger)
+            for page_data, page in pr_reviews_generator.iter_pages():
 
                 if page_data is None:
                     break
 
                 if len(page_data) == 0:
                     break
 
-                all_raw_pr_reviews.extend(page_data)
+                pr_reviews.extend(page_data)
+
+            if pr_reviews:
+                all_pr_reviews[pull_request_id] = pr_reviews
 
-        if not all_raw_pr_reviews:
+        if not list(all_pr_reviews.keys()):
             logger.info(f"{owner}/{repo} No pr reviews for repo")
             return
 
         contributors = []
-        for raw_pr_review in all_raw_pr_reviews:
-            contributor = process_pull_request_review_contributor(raw_pr_review, tool_source, tool_version, data_source)
-            if contributor:
-                contributors.append(contributor)
+        for pull_request_id in all_pr_reviews.keys():
+
+            reviews = all_pr_reviews[pull_request_id]
+            for review in reviews:
+                contributor = process_pull_request_review_contributor(review, tool_source, tool_version, data_source)
+                if contributor:
+                    contributors.append(contributor)
 
         logger.info(f"{owner}/{repo} Pr reviews: Inserting {len(contributors)} contributors")
         augur_db.insert_data(contributors, Contributor, ["cntrb_id"])
 
 
         pr_reviews = []
-        for raw_pr_review in all_raw_pr_reviews:
-
-            logger.info(f"Pr review type: {type(raw_pr_review)}")
-            logger.info(raw_pr_review)
+        for pull_request_id in all_pr_reviews.keys():
 
-            if "cntrb_id" in raw_pr_review:
-                pr_reviews.append(extract_needed_pr_review_data(raw_pr_review, pull_request_id, repo_id, platform_id, tool_source, tool_version))
+            reviews = all_pr_reviews[pull_request_id]
+            for review in reviews:
+
+                if "cntrb_id" in review:
+                    pr_reviews.append(extract_needed_pr_review_data(review, pull_request_id, repo_id, platform_id, tool_source, tool_version))
 
         logger.info(f"{owner}/{repo}: Inserting pr reviews of length: {len(pr_reviews)}")
         pr_review_natural_keys = ["pr_review_src_id",]
@@ -395,7 +401,3 @@ def collect_pull_request_reviews(repo_git: str) -> None:
 
 
 
-
-
-
-

augur/tasks/github/util/github_api_key_handler.py

@@ -76,8 +76,8 @@ def get_api_keys_from_database(self) -> List[str]:
         #select.order_by(func.random())
         where = [WorkerOauth.access_token != self.config_key, WorkerOauth.platform == 'github']
 
-        #return [key_tuple[0] for key_tuple in self.session.query(select).filter(*where).order_by(func.random()).all()]
-        return [key_tuple[0] for key_tuple in self.session.query(select).filter(*where).all()]
+        return [key_tuple[0] for key_tuple in self.session.query(select).filter(*where).order_by(func.random()).all()]
+        #return [key_tuple[0] for key_tuple in self.session.query(select).filter(*where).all()]
 
 
     def get_api_keys(self) -> List[str]:

augur/tasks/util/redis_list.py

@@ -170,8 +170,8 @@ def pop(self, index: int = None):
         if index is None:
             # This will get a random index from the list and remove it, 
             # decreasing the likelihood of everyone using the same key all the time
-            redis.rpop(self.redis_list_key)
-            #redis.spop(self.redis_list_key)
+            #redis.rpop(self.redis_list_key)
+            redis.spop(self.redis_list_key)
 
         else:
             # calls __delitem__

docker/backend/Dockerfile

@@ -2,7 +2,7 @@
 FROM python:3.8.11-slim-buster
 
 LABEL maintainer="[email protected]"
-LABEL version="0.52.0"
+LABEL version="0.53.1"
 
 ENV DEBIAN_FRONTEND=noninteractive
 

docker/database/Dockerfile

@@ -2,7 +2,7 @@
 FROM postgres:12
 
 LABEL maintainer="[email protected]"
-LABEL version="0.52.0"
+LABEL version="0.53.1"
 
 ENV POSTGRES_DB "test"
 ENV POSTGRES_USER "augur"

docs/dev-osx-install.md

@@ -1,5 +1,7 @@
 ## Augur Setup
 
+**NOTE**: Currently, our machine learning dependencies allow Augur to only fully support python 3.8 to python 3.10. Python 3.11 will sometimes work, but often there are libraries at the operating system level that have not yet been updated to support machine learning libraries at python 3.11. 
+
 # OSX: Note: This has **MOSTLY** been tested on Apple Silicon with Python 3.11 at this time, however, one user has been successful with Intel based Apple computers.
 ## For OSX You Need to make sure to install XCode Command line tools: 
 ```shell
@@ -38,6 +40,19 @@ export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig"
 ## Pre-Requisite Operating System Level Packages
 Here we ensure your system is up to date, install required python libraries, install postgresql, and install our queuing infrastrucutre, which is composed of redis-server and rabbitmq-server
 
+### Updating your Path: Necessary for rabbitmq on OSX
+#### for macOS Intel
+`export PATH=$PATH:/usr/local/sbin`
+#### for Apple Silicon
+`export PATH=$PATH:/opt/homebrew/sbin`
+
+***These should be added to your .zshrc or other environment file loaded when you open a terminal***
+
+#### for macOS Intel
+`export PATH=$PATH:/usr/local/sbin:$PATH`
+#### for Apple Silicon
+`export PATH=$PATH:/opt/homebrew/sbin:$PATH`
+
 ### Executable
 ```shell 
 brew update ; 
@@ -77,19 +92,6 @@ rabbitmqctl set_user_tags augur augurTag administrator;
 rabbitmqctl set_permissions -p augur_vhost augur ".*" ".*" ".*";
 ```
 
-### Updating your Path: Necessary for rabbitmq on OSX
-#### for macOS Intel
-`export PATH=$PATH:/usr/local/sbin`
-#### for Apple Silicon
-`export PATH=$PATH:/opt/homebrew/sbin`
-
-***These should be added to your .zshrc or other environment file loaded when you open a terminal***
-
-#### for macOS Intel
-`export PATH=$PATH:/usr/local/sbin:$PATH`
-#### for Apple Silicon
-`export PATH=$PATH:/opt/homebrew/sbin:$PATH`
-
 - We need rabbitmq_management so we can purge our own queues with an API call 
 - We need a user
 - We need a vhost