github api queries revised and refactored

sources/graphics_list_formatter.py

@@ -4,9 +4,7 @@ from datetime import datetime
 
 from pytz import timezone, utc
 
-from manager_download import DownloadManager as DM
 from manager_environment import EnvironmentManager as EM
-from manager_github import GitHubManager as GHM
 from manager_file import FileManager as FM
 
 
@@ -77,29 +75,25 @@ def make_list(data: List = None, names: List[str] = None, texts: List[str] = Non
     return "\n".join(data_list)
 
 
-async def make_commit_day_time_list(time_zone: str) -> str:
+async def make_commit_day_time_list(time_zone: str, repositories: Dict, commit_dates: Dict) -> str:
     """
     Calculate commit-related info, how many commits were made, and at what time of day and day of week.
 
     :param time_zone: User time zone.
+    :param repositories: User repositories list.
+    :param commit_dates: User commit data list.
     :returns: string representation of statistics.
     """
     stats = str()
-
-    result = await DM.get_remote_graphql("repos_contributed_to", username=GHM.USER.login)
-    repos = [d for d in result["data"]["user"]["repositoriesContributedTo"]["nodes"] if d["isFork"] is False]
-
     day_times = [0] * 4  # 0 - 6, 6 - 12, 12 - 18, 18 - 24
     week_days = [0] * 7  # Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday
 
-    for repository in repos:
+    for repository in [d for d in repositories["data"]["user"]["repositories"]["nodes"]]:
-        result = await DM.get_remote_graphql("repo_committed_dates", owner=repository["owner"]["login"], name=repository["name"], id=GHM.USER.node_id)
+        if repository["name"] not in commit_dates.keys():
-        if result["data"]["repository"] is None or result["data"]["repository"]["defaultBranchRef"] is None:
             continue
 
-        committed_dates = result["data"]["repository"]["defaultBranchRef"]["target"]["history"]["nodes"]
+        for committed_date in [commit_date for branch in commit_dates[repository["name"]].values() for commit_date in branch.values()]:
-        for committed_date in committed_dates:
+            local_date = datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%SZ")
-            local_date = datetime.strptime(committed_date["committedDate"], "%Y-%m-%dT%H:%M:%SZ")
             date = local_date.replace(tzinfo=utc).astimezone(timezone(time_zone))
 
             day_times[date.hour // 6] += 1

sources/main.py

@@ -3,6 +3,7 @@ Readme Development Metrics With waka time progress
 """
 from asyncio import run
 from datetime import datetime
+from typing import Dict
 from urllib.parse import quote
 
 from humanize import intword, naturalsize, intcomma
@@ -13,15 +14,17 @@ from manager_github import init_github_manager, GitHubManager as GHM
 from manager_file import init_localization_manager, FileManager as FM
 from manager_debug import init_debug_manager, DebugManager as DBM
 from graphics_chart_drawer import create_loc_graph, GRAPH_PATH
-from yearly_commit_calculator import calculate_yearly_commit_data
+from yearly_commit_calculator import calculate_commit_data
 from graphics_list_formatter import make_list, make_commit_day_time_list, make_language_per_repo_list
 
 
-async def get_waka_time_stats() -> str:
+async def get_waka_time_stats(repositories: Dict, commit_dates: Dict) -> str:
     """
     Collects user info from wakatime.
     Info includes most common commit time, timezone, language, editors, projects and OSs.
 
+    :param repositories: User repositories list.
+    :param commit_dates: User commit data list.
     :returns: String representation of the info.
     """
     DBM.i("Adding short WakaTime stats...")
@@ -30,7 +33,7 @@ async def get_waka_time_stats() -> str:
     data = await DM.get_remote_json("waka_latest")
     if EM.SHOW_COMMIT:
         DBM.i("Adding user commit day time info...")
-        stats += f"{await make_commit_day_time_list(data['data']['timezone'])}\n\n"
+        stats += f"{await make_commit_day_time_list(data['data']['timezone'], repositories, commit_dates)}\n\n"
 
     if EM.SHOW_TIMEZONE or EM.SHOW_LANGUAGE or EM.SHOW_EDITORS or EM.SHOW_PROJECTS or EM.SHOW_OS:
         no_activity = FM.t("No Activity Tracked This Week")
@@ -118,6 +121,25 @@ async def get_short_github_info() -> str:
     return stats
 
 
+async def collect_user_repositories() -> Dict:
+    """
+    Collects information about all the user repositories available.
+
+    :returns: Complete list of user repositories.
+    """
+    DBM.i("Getting user repositories list...")
+    repositories = await DM.get_remote_graphql("user_repository_list", username=GHM.USER.login, id=GHM.USER.node_id)
+    repo_names = [repo["name"] for repo in repositories["data"]["user"]["repositories"]["nodes"]]
+    DBM.g("\tUser repository list collected!")
+
+    contributed = await DM.get_remote_graphql("repos_contributed_to", username=GHM.USER.login)
+    contributed_nodes = [r for r in contributed["data"]["user"]["repositoriesContributedTo"]["nodes"] if r["name"] not in repo_names and not r["isFork"]]
+    DBM.g("\tUser contributed to repository list collected!")
+
+    repositories["data"]["user"]["repositories"]["nodes"] += contributed_nodes
+    return repositories
+
+
 async def get_stats() -> str:
     """
     Creates new README.md content from all the acquired statistics from all places.
@@ -128,12 +150,12 @@ async def get_stats() -> str:
     DBM.i("Collecting stats for README...")
 
     stats = str()
-    repositories = await DM.get_remote_graphql("user_repository_list", username=GHM.USER.login, id=GHM.USER.node_id)
+    repositories = await collect_user_repositories()
 
     if EM.SHOW_LINES_OF_CODE or EM.SHOW_LOC_CHART:
-        yearly_data = await calculate_yearly_commit_data(repositories)
+        yearly_data, commit_data = await calculate_commit_data(repositories)
     else:
-        yearly_data = (None, dict())
+        yearly_data, commit_data = dict(), dict()
         DBM.w("User yearly data not needed, skipped.")
 
     if EM.SHOW_TOTAL_CODE_TIME:
@@ -155,7 +177,7 @@ async def get_stats() -> str:
     if EM.SHOW_SHORT_INFO:
         stats += await get_short_github_info()
 
-    stats += await get_waka_time_stats()
+    stats += await get_waka_time_stats(repositories, commit_data)
 
     if EM.SHOW_LANGUAGE_PER_REPO:
         DBM.i("Adding language per repository info...")

sources/manager_download.py

@@ -18,11 +18,15 @@ GITHUB_API_QUERIES = {
     user(login: "$username") {
         repositoriesContributedTo(orderBy: {field: CREATED_AT, direction: DESC}, $pagination, includeUserRepositories: true) {
             nodes {
-                isFork
+                primaryLanguage {
+                    name
+                }
                 name
                 owner {
                     login
                 }
+                isPrivate
+                isFork
             }
             pageInfo {
                 endCursor
@@ -30,28 +34,6 @@ GITHUB_API_QUERIES = {
             }
         }
     }
-}""",
-    # Query to collect info about all commits in user repositories, including: commit date.
-    # NB! Query includes information about repositories owned by user only.
-    "repo_committed_dates": """
-{
-    repository(owner: "$owner", name: "$name") {
-        defaultBranchRef {
-            target {
-                ... on Commit {
-                    history($pagination, author: { id: "$id" }) {
-                        nodes {
-                            committedDate
-                        }
-                        pageInfo {
-                            endCursor
-                            hasNextPage
-                        }
-                    }
-                }
-            }
-        }
-    }
 }""",
     # Query to collect info about all repositories user created or collaborated on, including: name, primary language and owner login.
     # NB! Query doesn't include information about repositories user contributed to via pull requests.
@@ -106,6 +88,7 @@ GITHUB_API_QUERIES = {
                                 additions
                                 deletions
                                 committedDate
+                                oid
                             }
                         }
                         pageInfo {
@@ -119,6 +102,7 @@ GITHUB_API_QUERIES = {
     }
 }
 """,
+    # Query to hide outdated PR comment.
     "hide_outdated_comment": """
 mutation {
     minimizeComment(input: {classifier: OUTDATED, subjectId: "$id"}) {

sources/manager_file.py

@@ -1,7 +1,7 @@
 from os.path import join, isfile, dirname
 from pickle import load as load_pickle, dump as dump_pickle
 from json import load as load_json
-from typing import Dict, Optional
+from typing import Dict, Optional, Any
 
 from manager_environment import EnvironmentManager as EM
 
@@ -58,7 +58,7 @@ class FileManager:
             file.write(content)
 
     @staticmethod
-    def cache_binary(name: str, content: Optional[Dict] = None, assets: bool = False) -> Optional[Dict]:
+    def cache_binary(name: str, content: Optional[Any] = None, assets: bool = False) -> Optional[Any]:
         """
         Save binary output file if provided or read if content is None.
 

sources/yearly_commit_calculator.py

@@ -1,7 +1,7 @@
 from json import dumps
 from re import search
 from datetime import datetime
-from typing import Dict
+from typing import Dict, Tuple
 
 from manager_download import DownloadManager as DM
 from manager_environment import EnvironmentManager as EM
@@ -10,7 +10,7 @@ from manager_file import FileManager as FM
 from manager_debug import DebugManager as DBM
 
 
-async def calculate_yearly_commit_data(repositories: Dict) -> Dict:
+async def calculate_commit_data(repositories: Dict) -> Tuple[Dict, Dict]:
     """
     Calculate commit data by years.
     Commit data includes contribution additions and deletions in each quarter of each recorded year.
@@ -18,38 +18,40 @@ async def calculate_yearly_commit_data(repositories: Dict) -> Dict:
     :param repositories: user repositories info dictionary.
     :returns: Commit quarter yearly data dictionary.
     """
-    DBM.i("Calculating yearly commit data...")
+    DBM.i("Calculating commit data...")
     if EM.DEBUG_RUN:
-        content = FM.cache_binary("yearly_data.pick", assets=True)
+        content = FM.cache_binary("commits_data.pick", assets=True)
         if content is not None:
-            DBM.g("Yearly data restored from cache!")
+            DBM.g("Commit data restored from cache!")
-            return content
+            return tuple(content)
         else:
-            DBM.w("No cached yearly data found, recalculating...")
+            DBM.w("No cached commit data found, recalculating...")
 
     yearly_data = dict()
+    date_data = dict()
     total = len(repositories["data"]["user"]["repositories"]["nodes"])
     for ind, repo in enumerate(repositories["data"]["user"]["repositories"]["nodes"]):
         if repo["name"] not in EM.IGNORED_REPOS:
             repo_name = "[private]" if repo["isPrivate"] else f"{repo['owner']['login']}/{repo['name']}"
             DBM.i(f"\t{ind + 1}/{total} Retrieving repo: {repo_name}")
-            await update_yearly_data_with_commit_stats(repo, yearly_data)
+            await update_data_with_commit_stats(repo, yearly_data, date_data)
-    DBM.g("Yearly commit data calculated!")
+    DBM.g("Commit data calculated!")
 
     if EM.DEBUG_RUN:
-        FM.cache_binary("yearly_data.pick", yearly_data, assets=True)
+        FM.cache_binary("commits_data.pick", [yearly_data, date_data], assets=True)
-        FM.write_file("yearly_data.json", dumps(yearly_data), assets=True)
+        FM.write_file("commits_data.json", dumps([yearly_data, date_data]), assets=True)
-        DBM.g("Yearly data saved to cache!")
+        DBM.g("Commit data saved to cache!")
-    return yearly_data
+    return yearly_data, date_data
 
 
-async def update_yearly_data_with_commit_stats(repo_details: Dict, yearly_data: Dict):
+async def update_data_with_commit_stats(repo_details: Dict, yearly_data: Dict, date_data: Dict):
     """
     Updates yearly commit data with commits from given repository.
     Skips update if the commit isn't related to any repository.
 
     :param repo_details: Dictionary with information about the given repository.
     :param yearly_data: Yearly data dictionary to update.
+    :param date_data: Commit date dictionary to update.
     """
     owner = repo_details["owner"]["login"]
     branch_data = await DM.get_remote_graphql("repo_branch_list", owner=owner, name=repo_details["name"])
@@ -64,6 +66,12 @@ async def update_yearly_data_with_commit_stats(repo_details: Dict, yearly_data:
             curr_year = datetime.fromisoformat(date).year
             quarter = (datetime.fromisoformat(date).month - 1) // 3 + 1
 
+            if repo_details["name"] not in date_data:
+                date_data[repo_details["name"]] = dict()
+            if branch["name"] not in date_data[repo_details["name"]]:
+                date_data[repo_details["name"]][branch["name"]] = dict()
+            date_data[repo_details["name"]][branch["name"]][commit["oid"]] = commit["committedDate"]
+
             if repo_details["primaryLanguage"] is not None:
                 if curr_year not in yearly_data:
                     yearly_data[curr_year] = dict()