From 559b68bd8ba54e49021d31a9bd3868a7e2c61f93 Mon Sep 17 00:00:00 2001 From: pseusys Date: Sat, 11 Mar 2023 22:14:21 +0100 Subject: [PATCH 1/2] github api queries revised and refactored --- sources/graphics_list_formatter.py | 20 ++++++---------- sources/main.py | 36 +++++++++++++++++++++++------ sources/manager_download.py | 30 ++++++------------------ sources/manager_file.py | 4 ++-- sources/yearly_commit_calculator.py | 36 ++++++++++++++++++----------- 5 files changed, 67 insertions(+), 59 deletions(-) diff --git a/sources/graphics_list_formatter.py b/sources/graphics_list_formatter.py index fd06eb5..ce1263f 100644 --- a/sources/graphics_list_formatter.py +++ b/sources/graphics_list_formatter.py @@ -4,9 +4,7 @@ from datetime import datetime from pytz import timezone, utc -from manager_download import DownloadManager as DM from manager_environment import EnvironmentManager as EM -from manager_github import GitHubManager as GHM from manager_file import FileManager as FM @@ -77,29 +75,25 @@ def make_list(data: List = None, names: List[str] = None, texts: List[str] = Non return "\n".join(data_list) -async def make_commit_day_time_list(time_zone: str) -> str: +async def make_commit_day_time_list(time_zone: str, repositories: Dict, commit_dates: Dict) -> str: """ Calculate commit-related info, how many commits were made, and at what time of day and day of week. :param time_zone: User time zone. + :param repositories: User repositories list. + :param commit_dates: User commit data list. :returns: string representation of statistics. """ stats = str() - - result = await DM.get_remote_graphql("repos_contributed_to", username=GHM.USER.login) - repos = [d for d in result["data"]["user"]["repositoriesContributedTo"]["nodes"] if d["isFork"] is False] - day_times = [0] * 4 # 0 - 6, 6 - 12, 12 - 18, 18 - 24 week_days = [0] * 7 # Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday - for repository in repos: - result = await DM.get_remote_graphql("repo_committed_dates", owner=repository["owner"]["login"], name=repository["name"], id=GHM.USER.node_id) - if result["data"]["repository"] is None or result["data"]["repository"]["defaultBranchRef"] is None: + for repository in [d for d in repositories["data"]["user"]["repositories"]["nodes"]]: + if repository["name"] not in commit_dates.keys(): continue - committed_dates = result["data"]["repository"]["defaultBranchRef"]["target"]["history"]["nodes"] - for committed_date in committed_dates: - local_date = datetime.strptime(committed_date["committedDate"], "%Y-%m-%dT%H:%M:%SZ") + for committed_date in [commit_date for branch in commit_dates[repository["name"]].values() for commit_date in branch.values()]: + local_date = datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%SZ") date = local_date.replace(tzinfo=utc).astimezone(timezone(time_zone)) day_times[date.hour // 6] += 1 diff --git a/sources/main.py b/sources/main.py index ff571ee..a560f5c 100644 --- a/sources/main.py +++ b/sources/main.py @@ -3,6 +3,7 @@ Readme Development Metrics With waka time progress """ from asyncio import run from datetime import datetime +from typing import Dict from urllib.parse import quote from humanize import intword, naturalsize, intcomma @@ -13,15 +14,17 @@ from manager_github import init_github_manager, GitHubManager as GHM from manager_file import init_localization_manager, FileManager as FM from manager_debug import init_debug_manager, DebugManager as DBM from graphics_chart_drawer import create_loc_graph, GRAPH_PATH -from yearly_commit_calculator import calculate_yearly_commit_data +from yearly_commit_calculator import calculate_commit_data from graphics_list_formatter import make_list, make_commit_day_time_list, make_language_per_repo_list -async def get_waka_time_stats() -> str: +async def get_waka_time_stats(repositories: Dict, commit_dates: Dict) -> str: """ Collects user info from wakatime. Info includes most common commit time, timezone, language, editors, projects and OSs. + :param repositories: User repositories list. + :param commit_dates: User commit data list. :returns: String representation of the info. """ DBM.i("Adding short WakaTime stats...") @@ -30,7 +33,7 @@ async def get_waka_time_stats() -> str: data = await DM.get_remote_json("waka_latest") if EM.SHOW_COMMIT: DBM.i("Adding user commit day time info...") - stats += f"{await make_commit_day_time_list(data['data']['timezone'])}\n\n" + stats += f"{await make_commit_day_time_list(data['data']['timezone'], repositories, commit_dates)}\n\n" if EM.SHOW_TIMEZONE or EM.SHOW_LANGUAGE or EM.SHOW_EDITORS or EM.SHOW_PROJECTS or EM.SHOW_OS: no_activity = FM.t("No Activity Tracked This Week") @@ -118,6 +121,25 @@ async def get_short_github_info() -> str: return stats +async def collect_user_repositories() -> Dict: + """ + Collects information about all the user repositories available. + + :returns: Complete list of user repositories. + """ + DBM.i("Getting user repositories list...") + repositories = await DM.get_remote_graphql("user_repository_list", username=GHM.USER.login, id=GHM.USER.node_id) + repo_names = [repo["name"] for repo in repositories["data"]["user"]["repositories"]["nodes"]] + DBM.g("\tUser repository list collected!") + + contributed = await DM.get_remote_graphql("repos_contributed_to", username=GHM.USER.login) + contributed_nodes = [r for r in contributed["data"]["user"]["repositoriesContributedTo"]["nodes"] if r["name"] not in repo_names and not r["isFork"]] + DBM.g("\tUser contributed to repository list collected!") + + repositories["data"]["user"]["repositories"]["nodes"] += contributed_nodes + return repositories + + async def get_stats() -> str: """ Creates new README.md content from all the acquired statistics from all places. @@ -128,12 +150,12 @@ async def get_stats() -> str: DBM.i("Collecting stats for README...") stats = str() - repositories = await DM.get_remote_graphql("user_repository_list", username=GHM.USER.login, id=GHM.USER.node_id) + repositories = await collect_user_repositories() if EM.SHOW_LINES_OF_CODE or EM.SHOW_LOC_CHART: - yearly_data = await calculate_yearly_commit_data(repositories) + yearly_data, commit_data = await calculate_commit_data(repositories) else: - yearly_data = (None, dict()) + yearly_data, commit_data = dict(), dict() DBM.w("User yearly data not needed, skipped.") if EM.SHOW_TOTAL_CODE_TIME: @@ -155,7 +177,7 @@ async def get_stats() -> str: if EM.SHOW_SHORT_INFO: stats += await get_short_github_info() - stats += await get_waka_time_stats() + stats += await get_waka_time_stats(repositories, commit_data) if EM.SHOW_LANGUAGE_PER_REPO: DBM.i("Adding language per repository info...") diff --git a/sources/manager_download.py b/sources/manager_download.py index 79ef636..a90bf39 100644 --- a/sources/manager_download.py +++ b/sources/manager_download.py @@ -18,11 +18,15 @@ GITHUB_API_QUERIES = { user(login: "$username") { repositoriesContributedTo(orderBy: {field: CREATED_AT, direction: DESC}, $pagination, includeUserRepositories: true) { nodes { - isFork + primaryLanguage { + name + } name owner { login } + isPrivate + isFork } pageInfo { endCursor @@ -30,28 +34,6 @@ GITHUB_API_QUERIES = { } } } -}""", - # Query to collect info about all commits in user repositories, including: commit date. - # NB! Query includes information about repositories owned by user only. - "repo_committed_dates": """ -{ - repository(owner: "$owner", name: "$name") { - defaultBranchRef { - target { - ... on Commit { - history($pagination, author: { id: "$id" }) { - nodes { - committedDate - } - pageInfo { - endCursor - hasNextPage - } - } - } - } - } - } }""", # Query to collect info about all repositories user created or collaborated on, including: name, primary language and owner login. # NB! Query doesn't include information about repositories user contributed to via pull requests. @@ -106,6 +88,7 @@ GITHUB_API_QUERIES = { additions deletions committedDate + oid } } pageInfo { @@ -119,6 +102,7 @@ GITHUB_API_QUERIES = { } } """, + # Query to hide outdated PR comment. "hide_outdated_comment": """ mutation { minimizeComment(input: {classifier: OUTDATED, subjectId: "$id"}) { diff --git a/sources/manager_file.py b/sources/manager_file.py index 28041b6..4dc7c89 100644 --- a/sources/manager_file.py +++ b/sources/manager_file.py @@ -1,7 +1,7 @@ from os.path import join, isfile, dirname from pickle import load as load_pickle, dump as dump_pickle from json import load as load_json -from typing import Dict, Optional +from typing import Dict, Optional, Any from manager_environment import EnvironmentManager as EM @@ -58,7 +58,7 @@ class FileManager: file.write(content) @staticmethod - def cache_binary(name: str, content: Optional[Dict] = None, assets: bool = False) -> Optional[Dict]: + def cache_binary(name: str, content: Optional[Any] = None, assets: bool = False) -> Optional[Any]: """ Save binary output file if provided or read if content is None. diff --git a/sources/yearly_commit_calculator.py b/sources/yearly_commit_calculator.py index fb714da..10e2732 100644 --- a/sources/yearly_commit_calculator.py +++ b/sources/yearly_commit_calculator.py @@ -1,7 +1,7 @@ from json import dumps from re import search from datetime import datetime -from typing import Dict +from typing import Dict, Tuple from manager_download import DownloadManager as DM from manager_environment import EnvironmentManager as EM @@ -10,7 +10,7 @@ from manager_file import FileManager as FM from manager_debug import DebugManager as DBM -async def calculate_yearly_commit_data(repositories: Dict) -> Dict: +async def calculate_commit_data(repositories: Dict) -> Tuple[Dict, Dict]: """ Calculate commit data by years. Commit data includes contribution additions and deletions in each quarter of each recorded year. @@ -18,38 +18,40 @@ async def calculate_yearly_commit_data(repositories: Dict) -> Dict: :param repositories: user repositories info dictionary. :returns: Commit quarter yearly data dictionary. """ - DBM.i("Calculating yearly commit data...") + DBM.i("Calculating commit data...") if EM.DEBUG_RUN: - content = FM.cache_binary("yearly_data.pick", assets=True) + content = FM.cache_binary("commits_data.pick", assets=True) if content is not None: - DBM.g("Yearly data restored from cache!") - return content + DBM.g("Commit data restored from cache!") + return tuple(content) else: - DBM.w("No cached yearly data found, recalculating...") + DBM.w("No cached commit data found, recalculating...") yearly_data = dict() + date_data = dict() total = len(repositories["data"]["user"]["repositories"]["nodes"]) for ind, repo in enumerate(repositories["data"]["user"]["repositories"]["nodes"]): if repo["name"] not in EM.IGNORED_REPOS: repo_name = "[private]" if repo["isPrivate"] else f"{repo['owner']['login']}/{repo['name']}" DBM.i(f"\t{ind + 1}/{total} Retrieving repo: {repo_name}") - await update_yearly_data_with_commit_stats(repo, yearly_data) - DBM.g("Yearly commit data calculated!") + await update_data_with_commit_stats(repo, yearly_data, date_data) + DBM.g("Commit data calculated!") if EM.DEBUG_RUN: - FM.cache_binary("yearly_data.pick", yearly_data, assets=True) - FM.write_file("yearly_data.json", dumps(yearly_data), assets=True) - DBM.g("Yearly data saved to cache!") - return yearly_data + FM.cache_binary("commits_data.pick", [yearly_data, date_data], assets=True) + FM.write_file("commits_data.json", dumps([yearly_data, date_data]), assets=True) + DBM.g("Commit data saved to cache!") + return yearly_data, date_data -async def update_yearly_data_with_commit_stats(repo_details: Dict, yearly_data: Dict): +async def update_data_with_commit_stats(repo_details: Dict, yearly_data: Dict, date_data: Dict): """ Updates yearly commit data with commits from given repository. Skips update if the commit isn't related to any repository. :param repo_details: Dictionary with information about the given repository. :param yearly_data: Yearly data dictionary to update. + :param date_data: Commit date dictionary to update. """ owner = repo_details["owner"]["login"] branch_data = await DM.get_remote_graphql("repo_branch_list", owner=owner, name=repo_details["name"]) @@ -64,6 +66,12 @@ async def update_yearly_data_with_commit_stats(repo_details: Dict, yearly_data: curr_year = datetime.fromisoformat(date).year quarter = (datetime.fromisoformat(date).month - 1) // 3 + 1 + if repo_details["name"] not in date_data: + date_data[repo_details["name"]] = dict() + if branch["name"] not in date_data[repo_details["name"]]: + date_data[repo_details["name"]][branch["name"]] = dict() + date_data[repo_details["name"]][branch["name"]][commit["oid"]] = commit["committedDate"] + if repo_details["primaryLanguage"] is not None: if curr_year not in yearly_data: yearly_data[curr_year] = dict() From 4cea8cebbf66575ba649ee7376c21a256748a987 Mon Sep 17 00:00:00 2001 From: pseusys Date: Sat, 11 Mar 2023 22:26:13 +0100 Subject: [PATCH 2/2] implicit return fixed --- sources/yearly_commit_calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/yearly_commit_calculator.py b/sources/yearly_commit_calculator.py index 10e2732..3106db9 100644 --- a/sources/yearly_commit_calculator.py +++ b/sources/yearly_commit_calculator.py @@ -57,7 +57,7 @@ async def update_data_with_commit_stats(repo_details: Dict, yearly_data: Dict, d branch_data = await DM.get_remote_graphql("repo_branch_list", owner=owner, name=repo_details["name"]) if branch_data["data"]["repository"] is None: DBM.w(f"\t\tSkipping repo: {repo_details['name']}") - return dict() + return for branch in branch_data["data"]["repository"]["refs"]["nodes"]: commit_data = await DM.get_remote_graphql("repo_commit_list", owner=owner, name=repo_details["name"], branch=branch["name"], id=GHM.USER.node_id)