Skip to content

Infer bug_tracking_url and code_view_url from vcs_url #4944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kaushik-Kumar-CEG wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Infer bug_tracking_url and code_view_url from vcs_url #4944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/packagedcode/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -768,6 +768,8 @@ def from_data(cls, package_data, package_only=False):
else:
package_data.normalize_extracted_license_statement()

package_data.infer_vcs_urls()
Comment thread
Kaushik-Kumar-CEG marked this conversation as resolved.

return package_data

@property
Expand Down Expand Up @@ -857,6 +859,16 @@ def populate_license_fields(self):

self.normalize_extracted_license_statement()

def infer_vcs_urls(self):
from packagedcode.utils import parse_vcs_urls
if not self.vcs_url:
return
code_view, bug_track = parse_vcs_urls(self.vcs_url, version=self.version)
if not self.code_view_url and code_view:
self.code_view_url = code_view
if not self.bug_tracking_url and bug_track:
self.bug_tracking_url = bug_track

def update_purl_fields(self, package_data, replace=False):

if not self.type == package_data.type:
Expand Down
58 changes: 58 additions & 0 deletions src/packagedcode/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
# See https://aboutcode.org for more information about nexB OSS projects.
#

from urllib.parse import urlsplit

from packageurl import PackageURL

try:
Expand Down Expand Up @@ -114,6 +116,62 @@ def normalize_vcs_url(repo_url, vcs_tool=None):
return repo_url


BUG_TRACKING_SUFFIXES = {
'github.com': '/issues',
'gitlab.com': '/-/issues',
'codeberg.org': '/issues',
'bitbucket.org': '/issues',
}

# each platform has a different path scheme for browsing a tagged tree
CODE_VIEW_SUFFIXES = {
'github.com': '/tree/{version}',
'gitlab.com': '/-/tree/{version}',
'codeberg.org': '/src/tag/{version}',
'bitbucket.org': '/src/{version}',
}


def parse_vcs_urls(vcs_url, version=None):
"""
Given a ``vcs_url`` and an optional ``version``, return a
(code_view_url, bug_tracking_url) tuple for recognized hosting platforms,
or (None, None) otherwise.

``code_view_url`` points to the source tree at a specific version tag and
is only populated when a concrete ``version`` is provided.
"""
cleaned = normalize_vcs_url(vcs_url)
if not cleaned:
return None, None

if cleaned.endswith('.git'):
cleaned = cleaned[:-4]

# urlsplit can't parse git@ SSH form like git@github.com:owner/repo
if cleaned.startswith('git@'):
cleaned = 'https://' + cleaned[4:].replace(':', '/')

parsed = urlsplit(cleaned)
host = parsed.netloc.lower()
path = parsed.path.rstrip('/')
base = f'https://{host}{path}'

bug_suffix = BUG_TRACKING_SUFFIXES.get(host)
if not bug_suffix:
return None, None

bug_tracking_url = base + bug_suffix

code_view_url = None
if version:
code_suffix = CODE_VIEW_SUFFIXES.get(host)
if code_suffix:
code_view_url = base + code_suffix.format(version=version)

return code_view_url, bug_tracking_url


def build_description(summary, description):
"""
Return a description string from a summary and description
Expand Down
4 changes: 2 additions & 2 deletions tests/packagedcode/data/cargo/cargo_toml/clap/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/clap-rs/clap/issues",
"code_view_url": "https://github.com/clap-rs/clap/tree/2.32.0",
"vcs_url": "https://github.com/clap-rs/clap",
"copyright": null,
"holder": null,
Expand Down
4 changes: 2 additions & 2 deletions tests/packagedcode/data/cargo/cargo_toml/clippy/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/rust-lang/rust-clippy/issues",
"code_view_url": "https://github.com/rust-lang/rust-clippy/tree/0.0.212",
"vcs_url": "https://github.com/rust-lang/rust-clippy",
"copyright": null,
"holder": null,
Expand Down
4 changes: 2 additions & 2 deletions tests/packagedcode/data/cargo/cargo_toml/mdbook/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/rust-lang-nursery/mdBook/issues",
"code_view_url": "https://github.com/rust-lang-nursery/mdBook/tree/0.2.4-alpha.0",
"vcs_url": "https://github.com/rust-lang-nursery/mdBook",
"copyright": null,
"holder": null,
Expand Down
4 changes: 2 additions & 2 deletions tests/packagedcode/data/cargo/cargo_toml/rustfmt/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/rust-lang/rustfmt/issues",
"code_view_url": "https://github.com/rust-lang/rustfmt/tree/1.0.3",
"vcs_url": "https://github.com/rust-lang/rustfmt",
"copyright": null,
"holder": null,
Expand Down
4 changes: 2 additions & 2 deletions tests/packagedcode/data/cargo/cargo_toml/rustup/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/rust-lang/rustup.rs/issues",
"code_view_url": "https://github.com/rust-lang/rustup.rs/tree/1.17.0",
"vcs_url": "https://github.com/rust-lang/rustup.rs",
"copyright": null,
"holder": null,
Expand Down
6 changes: 3 additions & 3 deletions tests/packagedcode/data/cargo/cargo_toml/single-file-scan/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/cesarb/constant_time_eq/issues",
"code_view_url": "https://github.com/cesarb/constant_time_eq/tree/0.4.2",
"vcs_url": "https://github.com/cesarb/constant_time_eq",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -123,4 +123,4 @@
"scan_errors": []
}
]
}
}
8 changes: 4 additions & 4 deletions tests/packagedcode/data/cargo/scan-package-only.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,8 +327,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/indygreg/PyOxidizer/issues",
"code_view_url": "https://github.com/indygreg/PyOxidizer/tree/0.4.0-pre",
"vcs_url": "https://github.com/indygreg/PyOxidizer.git",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -669,8 +669,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/daac-tools/daachorse/issues",
"code_view_url": "https://github.com/daac-tools/daachorse/tree/0.4.1",
"vcs_url": "https://github.com/daac-tools/daachorse",
"copyright": null,
"holder": null,
Expand Down
16 changes: 8 additions & 8 deletions tests/packagedcode/data/cargo/scan.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/indygreg/PyOxidizer/issues",
"code_view_url": "https://github.com/indygreg/PyOxidizer/tree/0.4.0-pre",
"vcs_url": "https://github.com/indygreg/PyOxidizer.git",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -128,8 +128,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/daac-tools/daachorse/issues",
"code_view_url": "https://github.com/daac-tools/daachorse/tree/0.4.1",
"vcs_url": "https://github.com/daac-tools/daachorse",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -1173,8 +1173,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/indygreg/PyOxidizer/issues",
"code_view_url": "https://github.com/indygreg/PyOxidizer/tree/0.4.0-pre",
"vcs_url": "https://github.com/indygreg/PyOxidizer.git",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -1540,8 +1540,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/daac-tools/daachorse/issues",
"code_view_url": "https://github.com/daac-tools/daachorse/tree/0.4.1",
"vcs_url": "https://github.com/daac-tools/daachorse",
"copyright": null,
"holder": null,
Expand Down
8 changes: 4 additions & 4 deletions tests/packagedcode/data/conda/assembly-conda-scan.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
"md5": "8cc2fc3e2198c2efe6cd890a7684a16a",
"sha256": "940b9ae3f0b64e7ee51dfbdcfcffc674a447e5592f8a66d8064cd505fc122b78",
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/psf/requests/issues",
"code_view_url": "https://github.com/psf/requests/tree/2.32.3",
"vcs_url": "https://github.com/psf/requests",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -587,8 +587,8 @@
"md5": null,
"sha256": "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/psf/requests/issues",
"code_view_url": "https://github.com/psf/requests/tree/2.32.3",
"vcs_url": "https://github.com/psf/requests",
"copyright": null,
"holder": null,
Expand Down
4 changes: 2 additions & 2 deletions tests/packagedcode/data/conda/meta-yaml/abeona/meta.yaml-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
"md5": null,
"sha256": "bc7512f2eef785b037d836f4cc6faded457ac277f75c6e34eccd12da7c85258f",
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/winni2k/abeona/issues",
"code_view_url": "https://github.com/winni2k/abeona/tree/0.45.0",
"vcs_url": "https://github.com/winni2k/abeona",
"copyright": null,
"holder": null,
Expand Down
4 changes: 2 additions & 2 deletions tests/packagedcode/data/conda/meta-yaml/gcnvkernel/meta.yaml-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
"md5": null,
"sha256": "ac7015c3f0ef1852745ca0ef647adbf8ddef5db63ab485b00bc1ffe654814155",
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/broadinstitute/gatk/issues",
"code_view_url": "https://github.com/broadinstitute/gatk/tree/0.9",
"vcs_url": "https://github.com/broadinstitute/gatk",
"copyright": null,
"holder": null,
Expand Down
2 changes: 1 addition & 1 deletion tests/packagedcode/data/golang/gomod/kingpin/output.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"bug_tracking_url": "https://github.com/alecthomas/kingpin/issues",
"code_view_url": null,
"vcs_url": "https://github.com/alecthomas/kingpin.git",
"copyright": null,
Expand Down
2 changes: 1 addition & 1 deletion tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"bug_tracking_url": "https://github.com/census-instrumentation/opencensus-service/issues",
"code_view_url": null,
"vcs_url": "https://github.com/census-instrumentation/opencensus-service.git",
"copyright": null,
Expand Down
2 changes: 1 addition & 1 deletion tests/packagedcode/data/golang/gomod/participle/output.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"bug_tracking_url": "https://github.com/alecthomas/participle/issues",
"code_view_url": null,
"vcs_url": "https://github.com/alecthomas/participle.git",
"copyright": null,
Expand Down
2 changes: 1 addition & 1 deletion tests/packagedcode/data/golang/gomod/sample/output.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"bug_tracking_url": "https://github.com/alecthomas/sample/issues",
"code_view_url": null,
"vcs_url": "https://github.com/alecthomas/sample.git",
"copyright": null,
Expand Down
2 changes: 1 addition & 1 deletion tests/packagedcode/data/golang/gomod/uap-go/output.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"bug_tracking_url": "https://github.com/ua-parser/uap-go/issues",
"code_view_url": null,
"vcs_url": "https://github.com/ua-parser/uap-go.git",
"copyright": null,
Expand Down
2 changes: 1 addition & 1 deletion tests/packagedcode/data/golang/gomod/user_agent/output.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"bug_tracking_url": "https://github.com/mssola/user_agent/issues",
"code_view_url": null,
"vcs_url": "https://github.com/mssola/user_agent.git",
"copyright": null,
Expand Down
8 changes: 4 additions & 4 deletions ...data/license_detection/license-beside-manifest/google-built-collection-diag.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/google/built_collection.dart/issues",
"code_view_url": "https://github.com/google/built_collection.dart/tree/5.1.1",
"vcs_url": "https://github.com/google/built_collection.dart",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -209,8 +209,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/google/built_collection.dart/issues",
"code_view_url": "https://github.com/google/built_collection.dart/tree/5.1.1",
"vcs_url": "https://github.com/google/built_collection.dart",
"copyright": null,
"holder": null,
Expand Down
8 changes: 4 additions & 4 deletions ...code/data/license_detection/license-beside-manifest/google-built-collection.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/google/built_collection.dart/issues",
"code_view_url": "https://github.com/google/built_collection.dart/tree/5.1.1",
"vcs_url": "https://github.com/google/built_collection.dart",
"copyright": null,
"holder": null,
Expand Down Expand Up @@ -203,8 +203,8 @@
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"bug_tracking_url": "https://github.com/google/built_collection.dart/issues",
"code_view_url": "https://github.com/google/built_collection.dart/tree/5.1.1",
"vcs_url": "https://github.com/google/built_collection.dart",
"copyright": null,
"holder": null,
Expand Down
Loading
Loading