Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion packages/tangle-cli/src/tangle_cli/component_inspector.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from tangle_cli.handler import TangleCliHandler
from tangle_cli.models import ComponentInfo, ComponentSpec
from tangle_cli.utils import _normalize_git_url

if TYPE_CHECKING:
from tangle_cli.client import TangleApiClient
Expand Down Expand Up @@ -416,7 +417,8 @@ def transparency_check(spec: ComponentSpec) -> tuple[bool, str]:
if ann.get("git_remote_url") and (
ann.get("component_yaml_path") or ann.get("git_relative_dir")
):
return True, f"git source metadata links to {ann['git_remote_url']}"
safe_url = _normalize_git_url(ann["git_remote_url"])
return True, f"git source metadata links to {safe_url}"

impl = spec.implementation or {}
container = impl.get("container", {})
Expand Down Expand Up @@ -447,6 +449,10 @@ def _resolve_git_source(spec: ComponentSpec) -> dict[str, Any] | None:
if not git_url:
return None

# Server-supplied annotations may still carry credentials in the remote
# URL; strip them before building any browsable/emitted links.
git_url = _normalize_git_url(git_url)

sha = annotations.get("git_remote_sha", "")
branch = annotations.get("git_remote_branch", "main")
component_yaml = annotations.get("component_yaml_path")
Expand Down
10 changes: 9 additions & 1 deletion packages/tangle-cli/src/tangle_cli/component_publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,15 @@ def __init__(
self._git_root = str(git_root or git_info.get("_git_root") or "") or None
self.git_remote_sha = git_remote_sha or git_info.get("git_remote_sha")
self.git_remote_branch = git_remote_branch or git_info.get("git_remote_branch")
self.git_remote_url = git_remote_url or git_info.get("git_remote_url")
resolved_git_remote_url = git_remote_url or git_info.get("git_remote_url")
# Sanitize here as well as in get_git_info: a caller-supplied
# --git-remote-url bypasses get_git_info's normalization, so this is the
# single choke point that guarantees no credentials are persisted.
self.git_remote_url = (
utils._normalize_git_url(resolved_git_remote_url)
if resolved_git_remote_url
else resolved_git_remote_url
)
self.git_repo = git_repo

def _component_spec_model(self) -> type[Any]:
Expand Down
152 changes: 132 additions & 20 deletions packages/tangle-cli/src/tangle_cli/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -870,31 +870,143 @@ def normalize_annotation_paths(
_CI_REPO_URL_VARS: tuple[str, ...] = ("BUILDKITE_REPO", "GITHUB_SERVER_URL", "CI_REPOSITORY_URL")


def _normalize_git_url(url: str) -> str:
"""Normalize a git remote URL to a browsable HTTPS URL.
# Query-string parameter names that carry authentication material. Exact
# matches for short/ambiguous keys that must not be caught by the substring
# rules below (e.g. ``sig``, ``key``, ``auth``).
_SENSITIVE_QUERY_KEYS: frozenset[str] = frozenset({

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(AI-assisted)

Could we fail closed by dropping the query from normalized Git metadata, or reuse the broader sanitizer predicate?

Exact matching here misses common credential keys such as oauth_token and X-Amz-Signature, so their values can still reach the persisted/displayed URL despite the hard-guarantee claim. Since this normalized remote is used to build repository browse links, dropping the query entirely seems safest; alternatively, centralize the broader token/credential and signed-URL matching used in the related transport sanitizer work. Could we also add regression cases for these two keys?

"access_token", "personal_access_token", "private_token", "token",
"api_key", "apikey", "key", "auth", "authorization",
"password", "passwd", "pwd", "secret",
"sig", "signature", "x-access-token",
})

# Fail-closed substrings: any query key containing one of these (case-folded)
# is treated as credential-bearing even when it is not an exact known key. This
# catches provider-specific and future keys such as ``oauth_token``,
# ``X-Amz-Signature``, ``X-Amz-Credential`` or ``X-Amz-Security-Token`` without
# over-matching benign params like ``ref``/``path`` — which is why bare
# ``key``/``sig``/``auth`` stay exact-match only above.
_SENSITIVE_QUERY_SUBSTRINGS: tuple[str, ...] = (
"token", "secret", "password", "passwd", "credential",
"signature", "apikey", "api_key", "oauth", "x-amz-", "x-access",
)


def _is_sensitive_query_key(key: str) -> bool:
"""Return whether a URL query parameter name carries authentication material."""
folded = key.lower()
if folded in _SENSITIVE_QUERY_KEYS:
return True
return any(token in folded for token in _SENSITIVE_QUERY_SUBSTRINGS)

Handles common formats:
- ``git@github.com:Org/repo.git`` -> ``https://github.com/Org/repo``
- ``ssh://git@github.com/Org/repo.git`` -> ``https://github.com/Org/repo``
- ``https://github.com/Org/repo.git`` -> ``https://github.com/Org/repo``
- ``https://github.com/Org/repo`` -> unchanged

The ``.git`` suffix is stripped so the result can be used directly to
build ``/blob/{ref}/{path}`` links without an extra ``.removesuffix``.
def _redact_sensitive_query(query: str) -> str:
"""Drop credential-bearing parameters from a URL query string.

Uses a fail-closed predicate (:func:`_is_sensitive_query_key`) so unknown
credential-shaped keys are dropped rather than allowed through. A query
with no sensitive keys is returned byte-for-byte unchanged so that ordinary
URLs are not silently re-encoded.
"""
import re
if not query:
return query

from urllib.parse import parse_qsl, urlencode

pairs = parse_qsl(query, keep_blank_values=True)
if not any(_is_sensitive_query_key(key) for key, _ in pairs):
return query
kept = [(key, value) for key, value in pairs if not _is_sensitive_query_key(key)]
return urlencode(kept)


# Placeholder emitted when a URL-like remote carries credentials but cannot be
# parsed into a clean host (malformed authority, missing host with userinfo,
# malformed IPv6). Returning this rather than the raw input keeps credential
# material out of persisted annotations, CLI output, logs, and browse links, and
# stops the parser from raising into callers.
_REDACTED_GIT_URL: str = "[redacted-invalid-git-url]"

# SCP-style: git@host:path
m = re.match(r"^git@([^:]+):(.+)$", url)
if m:
url = f"https://{m.group(1)}/{m.group(2)}"
else:
# ssh://git@host/path
m = re.match(r"^ssh://(?:[^@]+@)?([^/]+)/(.+)$", url)
if m:
url = f"https://{m.group(1)}/{m.group(2)}"

return url.removesuffix(".git")
def _normalize_git_url(url: str) -> str:
"""Normalize a git remote URL to a browsable, credential-free HTTPS URL.

Converts SSH/SCP forms to HTTPS and strips the ``.git`` suffix so the
result can build ``/blob/{ref}/{path}`` links directly:

- ``git@github.com:Org/repo.git`` -> ``https://github.com/Org/repo``
- ``ssh://git@github.com/Org/repo.git`` -> ``https://github.com/Org/repo``
- ``https://github.com/Org/repo.git`` -> ``https://github.com/Org/repo``
- ``https://github.com/Org/repo`` -> unchanged

Any embedded credentials are removed: ``user:password@`` / ``token@``
userinfo is stripped from URL-form and scheme-relative remotes and dropped
entirely from SCP-style remotes, and sensitive query parameters are redacted.
This guarantees secrets never reach persisted annotations, CLI output, logs,
or error messages. Host, port, path, and fragment are preserved.

Parsing fails closed: a URL-like input whose credential-bearing authority
cannot be parsed into a clean host (missing host with userinfo, malformed
IPv6, and similar) yields ``_REDACTED_GIT_URL`` rather than leaking the raw
``user:secret@`` text or raising; a malformed textual port is dropped while
the credential-free host is kept. Local filesystem paths and hostless
schemes (e.g. ``file:///path``) are returned unchanged (aside from ``.git``
stripping). The function is idempotent.
"""
from urllib.parse import urlsplit, urlunsplit

if not url:
return url

stripped = url.strip()

# SCP-style syntax: ``[user@]host:path`` (no scheme). We drop any userinfo
# since it is authentication material, and rewrite to https so the result
# is browsable. Guard against Windows drive paths (``C:\...``) and against
# anything that already carries an explicit scheme.
if "://" not in stripped and not re.match(r"^[A-Za-z]:[\\/]", stripped):
scp = re.match(r"^(?:[^@/]+@)?([^/:]+):(?!//)(.+)$", stripped)
if scp:
stripped = f"https://{scp.group(1)}/{scp.group(2)}"

# URL-like inputs (explicit scheme or scheme-relative ``//authority``) must
# fail closed; a bare local path never leaks userinfo, so it is exempt.
url_like = "://" in stripped or stripped.startswith("//")

try:
parts = urlsplit(stripped)
host = parts.hostname
try:
port = parts.port
except ValueError:
# Malformed textual port (e.g. ``host:notaport``): drop the port but
# keep the credential-free host so the link stays browsable.
port = None
except ValueError:
# Malformed authority (e.g. unterminated IPv6 ``[::1``).
return _REDACTED_GIT_URL if url_like else stripped.removesuffix(".git")

if parts.scheme or parts.netloc:
if host is None:
# URL-form/scheme-relative with no parseable host. If the authority
# carried userinfo, returning the raw text would leak it — fail
# closed. Otherwise it is a legitimately hostless scheme (file://).
if "@" in parts.netloc:
return _REDACTED_GIT_URL
return stripped.removesuffix(".git")
# Re-bracket IPv6 literals, which ``hostname`` returns without brackets.
if ":" in host and not host.startswith("["):
host = f"[{host}]"
netloc = host if port is None else f"{host}:{port}"
out_scheme = "https" if parts.scheme.lower() == "ssh" else parts.scheme.lower()
query = _redact_sensitive_query(parts.query)
# Strip ``.git`` from the path itself so it is removed even when a query
# or fragment follows it.
path = parts.path.removesuffix(".git")
return urlunsplit((out_scheme, netloc, path, query, parts.fragment))

# No scheme and no authority: a bare local path.
return stripped.removesuffix(".git")


def _fill_from_ci_env(info: dict[str, str]) -> None:
Expand Down
21 changes: 21 additions & 0 deletions tests/test_component_inspector.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,27 @@ def test_unknown_container_is_opaque(self):
assert transparent is False
assert "no inline source" in reason

def test_git_source_reason_does_not_leak_credentials(self):
spec = ComponentSpec.from_dict({
"spec": {
"name": "demo",
"implementation": {"container": {"image": "registry.example.com/private/demo:latest"}},
"metadata": {
"annotations": {
"git_remote_url": "https://user:s3cr3tTOKEN@github.com/Org/repo.git",
"component_yaml_path": "comp.yaml",
},
},
},
})

transparent, reason = ComponentInspector.transparency_check(spec)

assert transparent is True
assert "s3cr3tTOKEN" not in reason
assert "@github.com" not in reason
assert "https://github.com/Org/repo" in reason


class TestComponentLibrary:
def test_standard_library_does_not_fetch_cross_origin_component_urls(self):
Expand Down
14 changes: 14 additions & 0 deletions tests/test_component_publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,20 @@ def test_publish_components_passes_structured_context_to_context_aware_hooks(tmp
assert kwargs_hook.contexts == [after_context]


def test_publisher_strips_credentials_from_supplied_git_remote_url(tmp_path: Path) -> None:
# A caller-supplied --git-remote-url bypasses get_git_info's normalization,
# so the publisher itself must sanitize it before it reaches annotations.
publisher = ComponentPublisher(
dry_run=True,
git_remote_url="https://gitlab-ci-token:glcbt-SECRETVALUE@gitlab.com/Org/repo.git",
git_root=tmp_path,
)

assert publisher.git_remote_url == "https://gitlab.com/Org/repo"
assert "glcbt-SECRETVALUE" not in (publisher.git_remote_url or "")
assert "@" not in (publisher.git_remote_url or "")


def test_publish_components_batches_configs_and_runs_hooks(tmp_path: Path) -> None:
first = write_component(tmp_path / "one.yaml", name="one", version="1.0")
second = write_component(tmp_path / "two.yaml", name="two", version="2.0")
Expand Down
Loading