pyOpenSourceProjects API Documentation

checkos

Created on 2024-07-30

@author: wf

CheckOS

check the open source projects

Source code in osprojects/checkos.py

class CheckOS:
    """
    check the open source projects
    """

    def __init__(self, args: Namespace, project: OsProject):
        self.args = args
        self.verbose = args.verbose
        self.workspace = args.workspace
        self.project = project
        self.project_path = os.path.join(self.workspace, project.id)
        self.checks = []
        # python 3.12 is max version
        self.max_python_version_minor=12

    @property
    def total(self) -> int:
        return len(self.checks)

    @property
    def ok_checks(self) -> List[Check]:
        ok_checks = [check for check in self.checks if check.ok]
        return ok_checks

    @property
    def failed_checks(self) -> List[Check]:
        failed_checks = [check for check in self.checks if not check.ok]
        return failed_checks

    def add_check(self, ok, msg:str="",path: str=None,negative:bool=False) -> Check:
        if not path:
            raise ValueError("path parameter missing")
        marker=""
        if negative:
            ok=not ok
            marker="⚠ ️"
        check = Check(ok=ok, path=path, msg=f"{marker}{msg}{path}")
        self.checks.append(check)
        return check

    def add_content_check(self, content: str, needle: str, path: str, negative:bool=False) -> Check:
        ok=needle in content
        check=self.add_check(ok, msg=f"{needle} in ", path=path,negative=negative)
        return check

    def add_path_check(self, path) -> Check:
        # Check if path exists
        path_exists = Check.file_exists(path)
        self.checks.append(path_exists)
        return path_exists

    def check_local(self) -> Check:
        local = Check.file_exists(self.project_path)
        return local

    def check_github_workflows(self):
        workflows_path = os.path.join(self.project_path, ".github", "workflows")
        workflows_exist = self.add_path_check(workflows_path)

        if workflows_exist.ok:
            required_files = ["build.yml", "upload-to-pypi.yml"]
            for file in required_files:
                file_path = os.path.join(workflows_path, file)
                file_exists = self.add_path_check(file_path)

                if file_exists.ok:
                    content = file_exists.content

                    if file == "build.yml":
                        min_python_version_minor = int(self.requires_python.split('.')[-1])
                        self.add_check(min_python_version_minor==self.min_python_version_minor,msg=f"{min_python_version_minor} (build.yml)!={self.min_python_version_minor} (pyprojec.toml)",path=file_path)
                        python_versions = f"""python-version: [ {', '.join([f"'3.{i}'" for i in range(self.min_python_version_minor, self.max_python_version_minor+1)])} ]"""
                        self.add_content_check(
                            content,
                            python_versions,
                            file_path,
                        )
                        self.add_content_check(
                            content,
                            "os: [ubuntu-latest, macos-latest, windows-latest]",
                            file_path,
                        )
                        self.add_content_check(content, "uses: actions/checkout@v4", file_path)
                        self.add_content_check(
                            content,
                            "uses: actions/setup-python@v5",
                            file_path,
                        )

                        self.add_content_check(
                            content,
                            "sphinx",
                            file_path,
                            negative=True
                        )
                        scripts_ok="scripts/install" in content and "scripts/test" in content or "scripts/installAndTest" in content
                        self.add_check(scripts_ok,"install and test", file_path)

                    elif file == "upload-to-pypi.yml":
                        self.add_content_check(content, "id-token: write", file_path)
                        self.add_content_check(content, "uses: actions/checkout@v4", file_path)
                        self.add_content_check(
                            content,
                            "uses: actions/setup-python@v5",
                            file_path,
                        )
                        self.add_content_check(
                            content,
                            "uses: pypa/gh-action-pypi-publish@release/v1",
                            file_path,
                        )

    def check_scripts(self):
        scripts_path = os.path.join(self.project_path, "scripts")
        scripts_exist = self.add_path_check(scripts_path)
        if scripts_exist.ok:
            required_files = ["blackisort", "test", "install", "doc", "release"]
            for file in required_files:
                file_path = os.path.join(scripts_path, file)
                file_exists = self.add_path_check(file_path)
                if file_exists.ok:
                    content = file_exists.content
                    if file=="doc":
                        self.add_content_check(content, "sphinx", file_path, negative=True)
                        self.add_content_check(content,"WF 2024-07-30 - updated",file_path)
                    if file=="test":
                        self.add_content_check(content,"WF 2024-08-03",file_path)
                    if file=="release":
                        self.add_content_check(content, "scripts/doc -d", file_path)

    def check_readme(self):
        readme_path = os.path.join(self.project_path, "README.md")
        readme_exists = self.add_path_check(readme_path)
        if readme_exists.ok:
            readme_content = readme_exists.content
            badge_lines = [
                "[![pypi](https://img.shields.io/pypi/pyversions/{self.project_name})](https://pypi.org/project/{self.project_name}/)",
                "[![Github Actions Build](https://github.com/{self.project.fqid}/actions/workflows/build.yml/badge.svg)](https://github.com/{self.project.fqid}/actions/workflows/build.yml)",
                "[![PyPI Status](https://img.shields.io/pypi/v/{self.project_name}.svg)](https://pypi.python.org/pypi/{self.project_name}/)",
                "[![GitHub issues](https://img.shields.io/github/issues/{self.project.fqid}.svg)](https://github.com/{self.project.fqid}/issues)",
                "[![GitHub closed issues](https://img.shields.io/github/issues-closed/{self.project.fqid}.svg)](https://github.com/{self.project.fqid}/issues/?q=is%3Aissue+is%3Aclosed)",
                "[![API Docs](https://img.shields.io/badge/API-Documentation-blue)](https://{self.project.owner}.github.io/{self.project.id}/)",
                "[![License](https://img.shields.io/github/license/{self.project.fqid}.svg)](https://www.apache.org/licenses/LICENSE-2.0)",
            ]
            for line in badge_lines:
                formatted_line = line.format(self=self)
                self.add_content_check(
                    content=readme_content,
                    needle=formatted_line,
                    path=readme_path,
                )
            self.add_content_check(readme_content, "readthedocs", readme_path, negative=True)

    def check_pyproject_toml(self):
        """
        pyproject.toml
        """
        toml_path = os.path.join(self.project_path, "pyproject.toml")
        toml_exists = self.add_path_check(toml_path)
        if toml_exists.ok:
            content=toml_exists.content
            toml_dict = tomllib.loads(content)
            project_check=self.add_check("project" in toml_dict, "[project]", toml_path)
            if project_check.ok:
                self.project_name=toml_dict["project"]["name"]
                requires_python_check=self.add_check("requires-python" in toml_dict["project"], "requires-python", toml_path)
                if requires_python_check.ok:
                    self.requires_python = toml_dict["project"]["requires-python"]
                    min_python_version = version.parse(self.requires_python.replace(">=", ""))
                    min_version_needed="3.9"
                    version_ok=min_python_version >= version.parse(min_version_needed)
                    self.add_check(version_ok, f"requires-python>={min_version_needed}", toml_path)
                    self.min_python_version_minor=int(str(min_python_version).split('.')[-1])
                    for minor_version in range(self.min_python_version_minor, self.max_python_version_minor+1):
                        needle=f"Programming Language :: Python :: 3.{minor_version}"
                        self.add_content_check(content, needle, toml_path)
            self.add_content_check(content, "hatchling", toml_path)
            self.add_content_check(content,"[tool.hatch.build.targets.wheel.sources]",toml_path)


    def check(self,title:str):
        """
        Check the given project and print results
        """
        self.check_local()
        self.check_pyproject_toml()
        self.check_github_workflows()
        self.check_readme()
        self.check_scripts()


        # ok_count=len(ok_checks)
        failed_count = len(self.failed_checks)
        summary = f"❌ {failed_count:2}/{self.total:2}" if failed_count > 0 else f"✅ {self.total:2}/{self.total:2}"
        print(f"{title}{summary}:{self.project}→{self.project.url}")
        if failed_count > 0:
            # Sort checks by path
            sorted_checks = sorted(self.checks, key=lambda c: c.path or "")

            # Group checks by path
            checks_by_path = {}
            for check in sorted_checks:
                if check.path not in checks_by_path:
                    checks_by_path[check.path] = []
                checks_by_path[check.path].append(check)

            # Display results
            for path, path_checks in checks_by_path.items():
                path_failed = sum(1 for c in path_checks if not c.ok)
                if path_failed > 0 or self.args.debug:
                    print(f"❌ {path}: {path_failed}")
                    i=0
                    for check in path_checks:
                        show=not check.ok or self.args.debug
                        if show:
                            i+=1
                            print(f"    {i:3}{check.marker}:{check.msg}")

                    if self.args.editor and path_failed > 0:
                        if os.path.isfile(path):
                            # @TODO Make editor configurable
                            Editor.open(path,default_editor_cmd="/usr/local/bin/atom")
                        else:
                            Editor.open_filepath(path)

check(title)

Check the given project and print results

Source code in osprojects/checkos.py

def check(self,title:str):
    """
    Check the given project and print results
    """
    self.check_local()
    self.check_pyproject_toml()
    self.check_github_workflows()
    self.check_readme()
    self.check_scripts()


    # ok_count=len(ok_checks)
    failed_count = len(self.failed_checks)
    summary = f"❌ {failed_count:2}/{self.total:2}" if failed_count > 0 else f"✅ {self.total:2}/{self.total:2}"
    print(f"{title}{summary}:{self.project}→{self.project.url}")
    if failed_count > 0:
        # Sort checks by path
        sorted_checks = sorted(self.checks, key=lambda c: c.path or "")

        # Group checks by path
        checks_by_path = {}
        for check in sorted_checks:
            if check.path not in checks_by_path:
                checks_by_path[check.path] = []
            checks_by_path[check.path].append(check)

        # Display results
        for path, path_checks in checks_by_path.items():
            path_failed = sum(1 for c in path_checks if not c.ok)
            if path_failed > 0 or self.args.debug:
                print(f"❌ {path}: {path_failed}")
                i=0
                for check in path_checks:
                    show=not check.ok or self.args.debug
                    if show:
                        i+=1
                        print(f"    {i:3}{check.marker}:{check.msg}")

                if self.args.editor and path_failed > 0:
                    if os.path.isfile(path):
                        # @TODO Make editor configurable
                        Editor.open(path,default_editor_cmd="/usr/local/bin/atom")
                    else:
                        Editor.open_filepath(path)

check_pyproject_toml()

pyproject.toml

Source code in osprojects/checkos.py

def check_pyproject_toml(self):
    """
    pyproject.toml
    """
    toml_path = os.path.join(self.project_path, "pyproject.toml")
    toml_exists = self.add_path_check(toml_path)
    if toml_exists.ok:
        content=toml_exists.content
        toml_dict = tomllib.loads(content)
        project_check=self.add_check("project" in toml_dict, "[project]", toml_path)
        if project_check.ok:
            self.project_name=toml_dict["project"]["name"]
            requires_python_check=self.add_check("requires-python" in toml_dict["project"], "requires-python", toml_path)
            if requires_python_check.ok:
                self.requires_python = toml_dict["project"]["requires-python"]
                min_python_version = version.parse(self.requires_python.replace(">=", ""))
                min_version_needed="3.9"
                version_ok=min_python_version >= version.parse(min_version_needed)
                self.add_check(version_ok, f"requires-python>={min_version_needed}", toml_path)
                self.min_python_version_minor=int(str(min_python_version).split('.')[-1])
                for minor_version in range(self.min_python_version_minor, self.max_python_version_minor+1):
                    needle=f"Programming Language :: Python :: 3.{minor_version}"
                    self.add_content_check(content, needle, toml_path)
        self.add_content_check(content, "hatchling", toml_path)
        self.add_content_check(content,"[tool.hatch.build.targets.wheel.sources]",toml_path)

main(_argv=None)

main command line entry point

Source code in osprojects/checkos.py

def main(_argv=None):
    """
    main command line entry point
    """
    parser = argparse.ArgumentParser(description="Check open source projects")
    parser.add_argument(
        "-d",
        "--debug",
        action="store_true",
        help="add debug output",
    )
    parser.add_argument(
        "-e",
        "--editor",
        action="store_true",
        help="open default editor on failed files",
    )
    parser.add_argument(
        "-o", "--owner", help="project owner or organization", required=True
    )
    parser.add_argument("-p", "--project", help="name of the project")
    parser.add_argument("-l", "--language", help="filter projects by language")
    parser.add_argument(
        "--local", action="store_true", help="check only locally available projects"
    )
    parser.add_argument(
        "-v", "--verbose", action="store_true", help="show verbose output"
    )
    parser.add_argument(
        "-ws",
        "--workspace",
        help="(Eclipse) workspace directory",
        default=os.path.expanduser("~/py-workspace"),
    )

    args = parser.parse_args(args=_argv)

    try:
        github = GitHub()
        if args.project:
            # Check specific project
            projects = github.list_projects_as_os_projects(
                args.owner, project_name=args.project
            )
        else:
            # Check all projects
            projects = github.list_projects_as_os_projects(args.owner)

        if args.language:
            projects = [p for p in projects if p.language == args.language]

        if args.local:
            local_projects = []
            for project in projects:
                checker = CheckOS(args=args, project=project)
                if checker.check_local().ok:
                    local_projects.append(project)
            projects = local_projects

        for i,project in enumerate(projects):
            checker = CheckOS(args=args, project=project)
            checker.check(f"{i+1:3}:")
    except Exception as ex:
        if args.debug:
            print(traceback.format_exc())
        raise ex

editor

Created on 2022-11-27

@author: wf

Editor

helper class to open the system defined editor

see https://stackoverflow.com/questions/1442841/lauch-default-editor-like-webbrowser-module

Source code in osprojects/editor.py

class Editor:
    """
    helper class to open the system defined editor

    see https://stackoverflow.com/questions/1442841/lauch-default-editor-like-webbrowser-module
    """

    @classmethod
    def open_filepath(cls, filepath: str):
        if platform.system() == "Darwin":  # macOS
            subprocess.call(("open", filepath))
        elif platform.system() == "Windows":  # Windows
            os.startfile(filepath, "open")
        else:  # linux variants
            subprocess.call(("xdg-open", filepath))

    @classmethod
    def extract_text(cls, html_text: str) -> str:
        """
        extract the text from the given html_text

        Args:
            html_text(str): the input for the html text

        Returns:
            str: the plain text
        """
        soup = BeautifulSoup(html_text, features="html.parser")

        # kill all script and style elements
        for script in soup(["script", "style"]):
            script.extract()  # rip it out

        # get text
        text = soup.get_text()

        # break into lines and remove leading and trailing space on each
        lines = (line.strip() for line in text.splitlines())
        # break multi-headlines into a line each
        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
        # drop blank lines
        text = "\n".join(chunk for chunk in chunks if chunk)
        return text

    @classmethod
    def open(
        cls,
        file_source: str,
        extract_text: bool = True,
        default_editor_cmd: str = "/usr/local/bin/atom",
    ) -> str:
        """
        open an editor for the given file_source

        Args:
            file_source(str): the path to the file
            extract_text(bool): if True extract the text from html sources

        Returns:
            str: the path to the file e.g. a temporary file if the file_source points to an url
        """
        # handle urls
        # https://stackoverflow.com/a/45886824/1497139
        if file_source.startswith("http"):
            url_source = urlopen(file_source)
            # https://stackoverflow.com/a/19156107/1497139
            charset = url_source.headers.get_content_charset()
            # if charset fails here you might want to set it to utf-8 as a default!
            text = url_source.read().decode(charset)
            if extract_text:
                # https://stackoverflow.com/a/24618186/1497139
                text = cls.extract_text(text)

            return cls.open_tmp_text(text)

        editor_cmd = None
        editor_env = os.getenv("EDITOR")
        if editor_env:
            editor_cmd = editor_env
        if platform.system() == "Darwin":
            if not editor_env:
                # https://stackoverflow.com/questions/22390709/how-can-i-open-the-atom-editor-from-the-command-line-in-os-x
                editor_cmd = default_editor_cmd
        if editor_cmd:
            os_cmd = f"{editor_cmd} {file_source}"
            os.system(os_cmd)
        return file_source

    @classmethod
    def open_tmp_text(cls, text: str, file_name: str = None) -> str:
        """
        open an editor for the given text in a newly created temporary file

        Args:
            text(str): the text to write to a temporary file and then open
            file_name(str): the name to use for the file

        Returns:
            str: the path to the temp file
        """
        # see https://stackoverflow.com/a/8577226/1497139
        # https://stackoverflow.com/a/3924253/1497139
        with tempfile.NamedTemporaryFile(delete=False) as tmp:
            with open(tmp.name, "w") as tmp_file:
                tmp_file.write(text)
                tmp_file.close()
            if file_name is None:
                file_path = tmp.name
            else:
                # https://stackoverflow.com/questions/3167154/how-to-split-a-dos-path-into-its-components-in-python
                path = Path(tmp.name)
                # https://stackoverflow.com/a/49798311/1497139
                file_path = path.parent / file_name
                os.rename(tmp.name, file_path)

            return cls.open(str(file_path))

extract_text(html_text) classmethod

extract the text from the given html_text

Parameters:

Name	Type	Description	Default
html_text(str)		the input for the html text	required

Returns:

Name	Type	Description
str	str	the plain text

Source code in osprojects/editor.py

@classmethod
def extract_text(cls, html_text: str) -> str:
    """
    extract the text from the given html_text

    Args:
        html_text(str): the input for the html text

    Returns:
        str: the plain text
    """
    soup = BeautifulSoup(html_text, features="html.parser")

    # kill all script and style elements
    for script in soup(["script", "style"]):
        script.extract()  # rip it out

    # get text
    text = soup.get_text()

    # break into lines and remove leading and trailing space on each
    lines = (line.strip() for line in text.splitlines())
    # break multi-headlines into a line each
    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
    # drop blank lines
    text = "\n".join(chunk for chunk in chunks if chunk)
    return text

open(file_source, extract_text=True, default_editor_cmd='/usr/local/bin/atom') classmethod

open an editor for the given file_source

Parameters:

Name	Type	Description	Default
file_source(str)		the path to the file	required
extract_text(bool)		if True extract the text from html sources	required

Returns:

Name	Type	Description
str	str	the path to the file e.g. a temporary file if the file_source points to an url

Source code in osprojects/editor.py

@classmethod
def open(
    cls,
    file_source: str,
    extract_text: bool = True,
    default_editor_cmd: str = "/usr/local/bin/atom",
) -> str:
    """
    open an editor for the given file_source

    Args:
        file_source(str): the path to the file
        extract_text(bool): if True extract the text from html sources

    Returns:
        str: the path to the file e.g. a temporary file if the file_source points to an url
    """
    # handle urls
    # https://stackoverflow.com/a/45886824/1497139
    if file_source.startswith("http"):
        url_source = urlopen(file_source)
        # https://stackoverflow.com/a/19156107/1497139
        charset = url_source.headers.get_content_charset()
        # if charset fails here you might want to set it to utf-8 as a default!
        text = url_source.read().decode(charset)
        if extract_text:
            # https://stackoverflow.com/a/24618186/1497139
            text = cls.extract_text(text)

        return cls.open_tmp_text(text)

    editor_cmd = None
    editor_env = os.getenv("EDITOR")
    if editor_env:
        editor_cmd = editor_env
    if platform.system() == "Darwin":
        if not editor_env:
            # https://stackoverflow.com/questions/22390709/how-can-i-open-the-atom-editor-from-the-command-line-in-os-x
            editor_cmd = default_editor_cmd
    if editor_cmd:
        os_cmd = f"{editor_cmd} {file_source}"
        os.system(os_cmd)
    return file_source

open_tmp_text(text, file_name=None) classmethod

open an editor for the given text in a newly created temporary file

Parameters:

Name	Type	Description	Default
text(str)		the text to write to a temporary file and then open	required
file_name(str)		the name to use for the file	required

Returns:

Name	Type	Description
str	str	the path to the temp file

Source code in osprojects/editor.py

@classmethod
def open_tmp_text(cls, text: str, file_name: str = None) -> str:
    """
    open an editor for the given text in a newly created temporary file

    Args:
        text(str): the text to write to a temporary file and then open
        file_name(str): the name to use for the file

    Returns:
        str: the path to the temp file
    """
    # see https://stackoverflow.com/a/8577226/1497139
    # https://stackoverflow.com/a/3924253/1497139
    with tempfile.NamedTemporaryFile(delete=False) as tmp:
        with open(tmp.name, "w") as tmp_file:
            tmp_file.write(text)
            tmp_file.close()
        if file_name is None:
            file_path = tmp.name
        else:
            # https://stackoverflow.com/questions/3167154/how-to-split-a-dos-path-into-its-components-in-python
            path = Path(tmp.name)
            # https://stackoverflow.com/a/49798311/1497139
            file_path = path.parent / file_name
            os.rename(tmp.name, file_path)

        return cls.open(str(file_path))

osproject

Created on 2022-01-24

@author: wf

Commit

Bases: object

a commit

Source code in osprojects/osproject.py

class Commit(object):
    """
    a commit
    """

    @staticmethod
    def getSamples():
        samples = [
            {
                "host": "https://github.com/WolfgangFahl/pyOpenSourceProjects",
                "path": "",
                "project": "pyOpenSourceProjects",
                "subject": "Initial commit",
                "name": "GitHub",  # TicketSystem
                "date": datetime.datetime.fromisoformat("2022-01-24 07:02:55+01:00"),
                "hash": "106254f",
            }
        ]
        return samples

    def toWikiMarkup(self):
        """
        Returns Commit as wiki markup
        """
        params = [
            f"{attr}={getattr(self, attr, '')}" for attr in self.getSamples()[0].keys()
        ]
        markup = f"{{{{commit|{'|'.join(params)}|storemode=subobject|viewmode=line}}}}"
        return markup

toWikiMarkup()

Returns Commit as wiki markup

Source code in osprojects/osproject.py

def toWikiMarkup(self):
    """
    Returns Commit as wiki markup
    """
    params = [
        f"{attr}={getattr(self, attr, '')}" for attr in self.getSamples()[0].keys()
    ]
    markup = f"{{{{commit|{'|'.join(params)}|storemode=subobject|viewmode=line}}}}"
    return markup

GitHub

Bases: TicketSystem

wrapper for the GitHub api

Source code in osprojects/osproject.py

class GitHub(TicketSystem):
    """
    wrapper for the GitHub api
    """

    @classmethod
    def load_access_token(cls) -> str:
        """
        if $HOME/.github/access_token.json exists read the token from there
        """
        # Specify the path to the access token file
        token_file_path = os.path.join(
            os.getenv("HOME"), ".github", "access_token.json"
        )

        # Check if the file exists and read the token
        if os.path.exists(token_file_path):
            with open(token_file_path, "r") as token_file:
                token_data = json.load(token_file)
                return token_data.get("access_token")

        # Return None if no token file is found
        return None

    @classmethod
    def prepare_headers(cls, access_token: str = None) -> dict:
        """
        Prepare authorization headers for GitHub API requests.
        """
        if access_token is None:
            access_token = cls.load_access_token()

        headers = {"Authorization": f"token {access_token}"} if access_token else {}
        return headers

    @classmethod
    def list_projects_as_os_projects(
        cls, owner: str, access_token: str = None, project_name: Optional[str] = None
    ) -> List[OsProject]:
        """
        List all public repositories for a given owner and return them as OsProject instances.

        Args:
            owner (str): The GitHub username or organization name.
            access_token (str, optional): GitHub personal access token for authentication.
            project_name (str, optional): If provided, return only this specific project.

        Returns:
            List[OsProject]: A list of OsProject instances representing the repositories.
        """
        headers = cls.prepare_headers(access_token)

        if project_name:
            url = f"https://api.github.com/repos/{owner}/{project_name}"
            response = requests.get(url, headers=headers)
            if response.status_code != 200:
                raise Exception(
                    f"Failed to fetch repository: {response.status_code} - {response.text}"
                )
            repos = [response.json()]
        else:
            url = f"https://api.github.com/users/{owner}/repos"
            params = {
                "type": "all",
                "per_page": 100,
            }  # Include all repo types, 100 per page
            all_repos = []
            page = 1

            while True:
                params["page"] = page
                response = requests.get(url, headers=headers, params=params)

                if response.status_code != 200:
                    raise Exception(
                        f"Failed to fetch repositories: {response.status_code} - {response.text}"
                    )

                repos = response.json()
                if not repos:
                    break  # No more repositories to fetch

                all_repos.extend(repos)
                page += 1

            repos = all_repos

        return [
            OsProject(
                owner=owner,
                id=repo["name"],
                ticketSystem=cls,
                title=repo["name"],
                url=repo["html_url"],
                description=repo["description"],
                language=repo["language"],
                created_at=datetime.datetime.fromisoformat(
                    repo["created_at"].rstrip("Z")
                ),
                updated_at=datetime.datetime.fromisoformat(
                    repo["updated_at"].rstrip("Z")
                ),
                stars=repo["stargazers_count"],
                forks=repo["forks_count"],
            )
            for repo in repos
        ]

    @classmethod
    def get_project(
        cls, owner: str, project_id: str, access_token: str = None
    ) -> OsProject:
        """
        Get a specific project as an OsProject instance.

        Args:
            owner (str): The GitHub username or organization name.
            project_id (str): The name of the project.
            access_token (str, optional): GitHub personal access token for authentication.

        Returns:
            OsProject: An OsProject instance representing the repository.
        """
        projects = cls.list_projects_as_os_projects(
            owner, access_token, project_name=project_id
        )
        if projects:
            return projects[0]
        raise Exception(f"Project {owner}/{project_id} not found")

    @classmethod
    def getIssues(
        cls, project: OsProject, access_token: str = None, limit: int = None, **params
    ) -> List[Ticket]:
        payload = {}
        headers = cls.prepare_headers(access_token)
        issues = []
        nextResults = True
        params["per_page"] = 100
        params["page"] = 1
        fetched_count = 0  # Counter to track the number of issues fetched
        while nextResults:
            response = requests.request(
                "GET",
                GitHub.ticketUrl(project),
                headers=headers,
                data=payload,
                params=params,
            )
            if response.status_code == 403 and "rate limit" in response.text:
                raise Exception("rate limit - you might want to use an access token")
            issue_records = json.loads(response.text)
            for record in issue_records:
                tr = {
                    "project": project,
                    "title": record.get("title"),
                    "body": record.get("body", ""),
                    "createdAt": (
                        parse(record.get("created_at"))
                        if record.get("created_at")
                        else ""
                    ),
                    "closedAt": (
                        parse(record.get("closed_at"))
                        if record.get("closed_at")
                        else ""
                    ),
                    "state": record.get("state"),
                    "number": record.get("number"),
                    "url": f"{cls.projectUrl(project)}/issues/{record.get('number')}",
                }
                issues.append(Ticket.init_from_dict(**tr))
                fetched_count += 1
                # Check if we have reached the limit
                if limit is not None and fetched_count >= limit:
                    nextResults = False
                    break

            if len(issue_records) < 100:
                nextResults = False
            else:
                params["page"] += 1
        return issues

    @classmethod
    def getComments(
        cls, project: OsProject, issue_number: int, access_token: str = None
    ) -> List[dict]:
        """
        Fetch all comments for a specific issue number from GitHub.
        """
        headers = cls.prepare_headers(access_token)
        comments_url = GitHub.commentUrl(project, issue_number)
        response = requests.get(comments_url, headers=headers)
        if response.status_code == 200:
            return response.json()
        else:
            raise Exception(
                f"Failed to fetch comments: {response.status_code} - {response.text}"
            )
        return []

    @staticmethod
    def projectUrl(project: OsProject):
        return f"https://github.com/{project.owner}/{project.id}"

    @staticmethod
    def ticketUrl(project: OsProject):
        return f"https://api.github.com/repos/{project.owner}/{project.id}/issues"

    @staticmethod
    def commitUrl(project: OsProject, id: str):
        return f"{GitHub.projectUrl(project)}/commit/{id}"

    @staticmethod
    def commentUrl(project: OsProject, issue_number: int):
        """
        Construct the URL for accessing comments of a specific issue.
        """
        return f"https://api.github.com/repos/{project.owner}/{project.id}/issues/{issue_number}/comments"

    @staticmethod
    def resolveProjectUrl(url: str) -> (str, str):
        """
        Resolve project url to owner and project name

        Returns:
            (owner, project)
        """
        # https://www.rfc-editor.org/rfc/rfc3986#appendix-B
        pattern = r"((https?:\/\/github\.com\/)|(git@github\.com:))(?P<owner>[^/?#]+)\/(?P<project>[^\./?#]+)(\.git)?"
        match = re.match(pattern=pattern, string=url)
        owner = match.group("owner")
        project = match.group("project")
        if owner and project:
            return owner, project

commentUrl(project, issue_number) staticmethod

Construct the URL for accessing comments of a specific issue.

Source code in osprojects/osproject.py

@staticmethod
def commentUrl(project: OsProject, issue_number: int):
    """
    Construct the URL for accessing comments of a specific issue.
    """
    return f"https://api.github.com/repos/{project.owner}/{project.id}/issues/{issue_number}/comments"

getComments(project, issue_number, access_token=None) classmethod

Fetch all comments for a specific issue number from GitHub.

Source code in osprojects/osproject.py

@classmethod
def getComments(
    cls, project: OsProject, issue_number: int, access_token: str = None
) -> List[dict]:
    """
    Fetch all comments for a specific issue number from GitHub.
    """
    headers = cls.prepare_headers(access_token)
    comments_url = GitHub.commentUrl(project, issue_number)
    response = requests.get(comments_url, headers=headers)
    if response.status_code == 200:
        return response.json()
    else:
        raise Exception(
            f"Failed to fetch comments: {response.status_code} - {response.text}"
        )
    return []

get_project(owner, project_id, access_token=None) classmethod

Get a specific project as an OsProject instance.

Parameters:

Name	Type	Description	Default
owner	str	The GitHub username or organization name.	required
project_id	str	The name of the project.	required
access_token	str	GitHub personal access token for authentication.	None

Returns:

Name	Type	Description
OsProject	OsProject	An OsProject instance representing the repository.

Source code in osprojects/osproject.py

@classmethod
def get_project(
    cls, owner: str, project_id: str, access_token: str = None
) -> OsProject:
    """
    Get a specific project as an OsProject instance.

    Args:
        owner (str): The GitHub username or organization name.
        project_id (str): The name of the project.
        access_token (str, optional): GitHub personal access token for authentication.

    Returns:
        OsProject: An OsProject instance representing the repository.
    """
    projects = cls.list_projects_as_os_projects(
        owner, access_token, project_name=project_id
    )
    if projects:
        return projects[0]
    raise Exception(f"Project {owner}/{project_id} not found")

list_projects_as_os_projects(owner, access_token=None, project_name=None) classmethod

List all public repositories for a given owner and return them as OsProject instances.

Parameters:

Name	Type	Description	Default
owner	str	The GitHub username or organization name.	required
access_token	str	GitHub personal access token for authentication.	None
project_name	str	If provided, return only this specific project.	None

Returns:

Type	Description
List[OsProject]	List[OsProject]: A list of OsProject instances representing the repositories.

Source code in osprojects/osproject.py

@classmethod
def list_projects_as_os_projects(
    cls, owner: str, access_token: str = None, project_name: Optional[str] = None
) -> List[OsProject]:
    """
    List all public repositories for a given owner and return them as OsProject instances.

    Args:
        owner (str): The GitHub username or organization name.
        access_token (str, optional): GitHub personal access token for authentication.
        project_name (str, optional): If provided, return only this specific project.

    Returns:
        List[OsProject]: A list of OsProject instances representing the repositories.
    """
    headers = cls.prepare_headers(access_token)

    if project_name:
        url = f"https://api.github.com/repos/{owner}/{project_name}"
        response = requests.get(url, headers=headers)
        if response.status_code != 200:
            raise Exception(
                f"Failed to fetch repository: {response.status_code} - {response.text}"
            )
        repos = [response.json()]
    else:
        url = f"https://api.github.com/users/{owner}/repos"
        params = {
            "type": "all",
            "per_page": 100,
        }  # Include all repo types, 100 per page
        all_repos = []
        page = 1

        while True:
            params["page"] = page
            response = requests.get(url, headers=headers, params=params)

            if response.status_code != 200:
                raise Exception(
                    f"Failed to fetch repositories: {response.status_code} - {response.text}"
                )

            repos = response.json()
            if not repos:
                break  # No more repositories to fetch

            all_repos.extend(repos)
            page += 1

        repos = all_repos

    return [
        OsProject(
            owner=owner,
            id=repo["name"],
            ticketSystem=cls,
            title=repo["name"],
            url=repo["html_url"],
            description=repo["description"],
            language=repo["language"],
            created_at=datetime.datetime.fromisoformat(
                repo["created_at"].rstrip("Z")
            ),
            updated_at=datetime.datetime.fromisoformat(
                repo["updated_at"].rstrip("Z")
            ),
            stars=repo["stargazers_count"],
            forks=repo["forks_count"],
        )
        for repo in repos
    ]

load_access_token() classmethod

if $HOME/.github/access_token.json exists read the token from there

Source code in osprojects/osproject.py

@classmethod
def load_access_token(cls) -> str:
    """
    if $HOME/.github/access_token.json exists read the token from there
    """
    # Specify the path to the access token file
    token_file_path = os.path.join(
        os.getenv("HOME"), ".github", "access_token.json"
    )

    # Check if the file exists and read the token
    if os.path.exists(token_file_path):
        with open(token_file_path, "r") as token_file:
            token_data = json.load(token_file)
            return token_data.get("access_token")

    # Return None if no token file is found
    return None

prepare_headers(access_token=None) classmethod

Prepare authorization headers for GitHub API requests.

Source code in osprojects/osproject.py

@classmethod
def prepare_headers(cls, access_token: str = None) -> dict:
    """
    Prepare authorization headers for GitHub API requests.
    """
    if access_token is None:
        access_token = cls.load_access_token()

    headers = {"Authorization": f"token {access_token}"} if access_token else {}
    return headers

resolveProjectUrl(url) staticmethod

Resolve project url to owner and project name

Returns:

Type	Description
(str, str)	(owner, project)

Source code in osprojects/osproject.py

@staticmethod
def resolveProjectUrl(url: str) -> (str, str):
    """
    Resolve project url to owner and project name

    Returns:
        (owner, project)
    """
    # https://www.rfc-editor.org/rfc/rfc3986#appendix-B
    pattern = r"((https?:\/\/github\.com\/)|(git@github\.com:))(?P<owner>[^/?#]+)\/(?P<project>[^\./?#]+)(\.git)?"
    match = re.match(pattern=pattern, string=url)
    owner = match.group("owner")
    project = match.group("project")
    if owner and project:
        return owner, project

Jira

Bases: TicketSystem

wrapper for Jira api

Source code in osprojects/osproject.py

class Jira(TicketSystem):
    """
    wrapper for Jira api
    """

OsProject

Bases: object

an Open Source Project

Source code in osprojects/osproject.py

class OsProject(object):
    """
    an Open Source Project
    """

    def __init__(
        self,
        owner: str = None,
        id: str = None,
        ticketSystem: Type[TicketSystem] = None,
        title: str = None,
        url: str = None,
        description: str = None,
        language: str = None,
        created_at: datetime.datetime = None,
        updated_at: datetime.datetime = None,
        stars: int = 0,
        forks: int = 0,
    ):
        """
        Constructor
        """
        self.owner = owner
        self.id = id
        self.ticketSystem = ticketSystem or GitHub
        self.title = title
        self.url = url
        self.description = description
        self.language = language
        self.created_at = created_at
        self.updated_at = updated_at
        self.stars = stars
        self.forks = forks

    @property
    def fqid(self):
        fqid = f"{self.owner}/{self.id}"
        return fqid

    def __str__(self):
        return self.fqid

    @staticmethod
    def getSamples():
        samples = [
            {
                "id": "pyOpenSourceProjects",
                "owner": "WolfgangFahl",
                "title": "pyOpenSourceProjects",
                "url": "https://github.com/WolfgangFahl/pyOpenSourceProjects",
                "description": "Helper Library to organize open source Projects",
                "language": "Python",
                "created_at": datetime.datetime(year=2022, month=1, day=24),
                "updated_at": datetime.datetime(year=2022, month=1, day=24),
                "stars": 5,
                "forks": 2,
            }
        ]
        return samples

    @classmethod
    def fromRepo(cls):
        """
        Init OsProject from repo in current working directory
        """
        url = subprocess.check_output(["git", "config", "--get", "remote.origin.url"])
        url = url.decode().strip("\n")
        return cls.fromUrl(url)

    @classmethod
    def fromUrl(cls, url: str) -> OsProject:
        """
        Init OsProject from given url
        """
        if "github.com" in url:
            owner, project_id = GitHub.resolveProjectUrl(url)
            if owner and project_id:
                github = GitHub()
                project = github.get_project(owner, project_id)
                return project
        raise Exception(f"Could not resolve the url '{url}' to a OsProject object")

    def getIssues(self, **params) -> list:
        tickets = self.ticketSystem.getIssues(self, **params)
        tickets.sort(key=lambda r: getattr(r, "number"), reverse=True)
        return tickets

    def getAllTickets(self, limit: int = None, **params):
        """
        Get all Tickets of the project -  closed and open ones

        Args:
            limit(int): if set limit the number of tickets retrieved
        """
        issues = self.getIssues(state="all", limit=limit, **params)
        return issues

    def getCommits(self) -> List[Commit]:
        commits = []
        gitlogCmd = [
            "git",
            "--no-pager",
            "log",
            "--reverse",
            r'--pretty=format:{"name":"%cn","date":"%cI","hash":"%h"}',
        ]
        gitLogCommitSubject = ["git", "log", "--format=%s", "-n", "1"]
        rawCommitLogs = subprocess.check_output(gitlogCmd).decode()
        for rawLog in rawCommitLogs.split("\n"):
            log = json.loads(rawLog)
            if log.get("date", None) is not None:
                log["date"] = datetime.datetime.fromisoformat(log["date"])
            log["project"] = self.id
            log["host"] = self.ticketSystem.projectUrl(self)
            log["path"] = ""
            log["subject"] = subprocess.check_output(
                [*gitLogCommitSubject, log["hash"]]
            )[
                :-1
            ].decode()  # seperate query to avoid json escaping issues
            commit = Commit()
            for k, v in log.items():
                setattr(commit, k, v)
            commits.append(commit)
        return commits

__init__(owner=None, id=None, ticketSystem=None, title=None, url=None, description=None, language=None, created_at=None, updated_at=None, stars=0, forks=0)

Constructor

Source code in osprojects/osproject.py

def __init__(
    self,
    owner: str = None,
    id: str = None,
    ticketSystem: Type[TicketSystem] = None,
    title: str = None,
    url: str = None,
    description: str = None,
    language: str = None,
    created_at: datetime.datetime = None,
    updated_at: datetime.datetime = None,
    stars: int = 0,
    forks: int = 0,
):
    """
    Constructor
    """
    self.owner = owner
    self.id = id
    self.ticketSystem = ticketSystem or GitHub
    self.title = title
    self.url = url
    self.description = description
    self.language = language
    self.created_at = created_at
    self.updated_at = updated_at
    self.stars = stars
    self.forks = forks

fromRepo() classmethod

Init OsProject from repo in current working directory

Source code in osprojects/osproject.py

@classmethod
def fromRepo(cls):
    """
    Init OsProject from repo in current working directory
    """
    url = subprocess.check_output(["git", "config", "--get", "remote.origin.url"])
    url = url.decode().strip("\n")
    return cls.fromUrl(url)

fromUrl(url) classmethod

Init OsProject from given url

Source code in osprojects/osproject.py

@classmethod
def fromUrl(cls, url: str) -> OsProject:
    """
    Init OsProject from given url
    """
    if "github.com" in url:
        owner, project_id = GitHub.resolveProjectUrl(url)
        if owner and project_id:
            github = GitHub()
            project = github.get_project(owner, project_id)
            return project
    raise Exception(f"Could not resolve the url '{url}' to a OsProject object")

getAllTickets(limit=None, **params)

Get all Tickets of the project - closed and open ones

Parameters:

Name	Type	Description	Default
limit(int)		if set limit the number of tickets retrieved	required

Source code in osprojects/osproject.py

def getAllTickets(self, limit: int = None, **params):
    """
    Get all Tickets of the project -  closed and open ones

    Args:
        limit(int): if set limit the number of tickets retrieved
    """
    issues = self.getIssues(state="all", limit=limit, **params)
    return issues

Ticket

Bases: object

a Ticket

Source code in osprojects/osproject.py

class Ticket(object):
    """
    a Ticket
    """

    @staticmethod
    def getSamples():
        samples = [
            {
                "number": 2,
                "title": "Get Tickets in Wiki notation from github API",
                "createdAt": datetime.datetime.fromisoformat(
                    "2022-01-24 07:41:29+00:00"
                ),
                "closedAt": datetime.datetime.fromisoformat(
                    "2022-01-25 07:43:04+00:00"
                ),
                "url": "https://github.com/WolfgangFahl/pyOpenSourceProjects/issues/2",
                "project": "pyOpenSourceProjects",
                "state": "closed",
            }
        ]
        return samples

    @classmethod
    def init_from_dict(cls, **records):
        """
        inits Ticket from given args
        """
        issue = Ticket()
        for k, v in records.items():
            setattr(issue, k, v)
        return issue

    def toWikiMarkup(self) -> str:
        """
        Returns Ticket in wiki markup
        """
        return f"""# {{{{Ticket
|number={self.number}
|title={self.title}
|project={self.project.id}
|createdAt={self.createdAt if self.createdAt else ""}
|closedAt={self.closedAt if self.closedAt else ""}
|state={self.state}
}}}}"""

init_from_dict(**records) classmethod

inits Ticket from given args

Source code in osprojects/osproject.py

@classmethod
def init_from_dict(cls, **records):
    """
    inits Ticket from given args
    """
    issue = Ticket()
    for k, v in records.items():
        setattr(issue, k, v)
    return issue

toWikiMarkup()

Returns Ticket in wiki markup

Source code in osprojects/osproject.py

    def toWikiMarkup(self) -> str:
        """
        Returns Ticket in wiki markup
        """
        return f"""# {{{{Ticket
|number={self.number}
|title={self.title}
|project={self.project.id}
|createdAt={self.createdAt if self.createdAt else ""}
|closedAt={self.closedAt if self.closedAt else ""}
|state={self.state}
}}}}"""

TicketSystem

Bases: object

platform for hosting OpenSourceProjects and their issues

Source code in osprojects/osproject.py

class TicketSystem(object):
    """
    platform for hosting OpenSourceProjects and their issues
    """

    @classmethod
    def getIssues(self, project: OsProject, **kwargs) -> List[Ticket]:
        """
        get issues from the TicketSystem for a project
        """
        return NotImplemented

    @staticmethod
    def projectUrl(project: OsProject):
        """
        url of the project
        """
        return NotImplemented

    @staticmethod
    def ticketUrl(project: OsProject):
        """
        url of the ticket/issue list
        """
        return NotImplemented

    @staticmethod
    def commitUrl(project: OsProject, id: str):
        """
        url of the ticket/issue list
        """
        return NotImplemented

commitUrl(project, id) staticmethod

url of the ticket/issue list

Source code in osprojects/osproject.py

@staticmethod
def commitUrl(project: OsProject, id: str):
    """
    url of the ticket/issue list
    """
    return NotImplemented

getIssues(project, **kwargs) classmethod

get issues from the TicketSystem for a project

Source code in osprojects/osproject.py

@classmethod
def getIssues(self, project: OsProject, **kwargs) -> List[Ticket]:
    """
    get issues from the TicketSystem for a project
    """
    return NotImplemented

projectUrl(project) staticmethod

url of the project

Source code in osprojects/osproject.py

@staticmethod
def projectUrl(project: OsProject):
    """
    url of the project
    """
    return NotImplemented

ticketUrl(project) staticmethod

url of the ticket/issue list

Source code in osprojects/osproject.py

@staticmethod
def ticketUrl(project: OsProject):
    """
    url of the ticket/issue list
    """
    return NotImplemented

gitlog2wiki(_argv=None)

cmdline interface to get gitlog entries in wiki markup

Source code in osprojects/osproject.py

def gitlog2wiki(_argv=None):
    """
    cmdline interface to get gitlog entries in wiki markup
    """
    parser = argparse.ArgumentParser(description="gitlog2wiki")
    if _argv:
        _args = parser.parse_args(args=_argv)

    osProject = OsProject.fromRepo()
    commits = osProject.getCommits()
    print("\n".join([c.toWikiMarkup() for c in commits]))

main(_argv=None)

main command line entry point

Source code in osprojects/osproject.py

def main(_argv=None):
    """
    main command line entry point
    """
    parser = argparse.ArgumentParser(description="Issue2ticket")
    parser.add_argument("-o", "--owner", help="project owner or organization")
    parser.add_argument("-p", "--project", help="name of the project")
    parser.add_argument(
        "--repo",
        action="store_true",
        help="get needed information form repository of current location",
    )
    parser.add_argument(
        "-ts",
        "--ticketsystem",
        default="github",
        choices=["github", "jira"],
        help="platform the project is hosted",
    )
    parser.add_argument(
        "-s",
        "--state",
        choices=["open", "closed", "all"],
        default="all",
        help="only issues with the given state",
    )
    parser.add_argument("-V", "--version", action="version", version="gitlog2wiki 0.1")

    args = parser.parse_args(args=_argv)
    # resolve ticketsystem
    ticketSystem = GitHub
    if args.ticketsystem == "jira":
        ticketSystem = Jira
    if args.project and args.owner:
        osProject = OsProject(
            owner=args.owner, id=args.project, ticketSystem=ticketSystem
        )
    else:
        osProject = OsProject.fromRepo()
    tickets = osProject.getIssues(state=args.state)
    print("\n".join([t.toWikiMarkup() for t in tickets]))