Skip to content

Commit

Permalink
Merge pull request #107 from gasinvein/cargo-fixes
Browse files Browse the repository at this point in the history 
cargo: Partial rewrite
  • Loading branch information
@gasinvein
gasinvein authored Apr 12, 2020
2 parents 062ca7e + 057e3a1 commit dd49b1a
Showing 1 changed file with 93 additions and 98 deletions.
191 changes: 93 additions & 98 deletions cargo/flatpak-cargo-generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import json
from urllib.parse import quote as urlquote
from urllib.parse import urlparse, ParseResult, parse_qs
import sys
import argparse
import logging

Expand All @@ -28,32 +27,33 @@
CARGO_HOME = 'cargo'
CARGO_GIT_DB = f'{CARGO_HOME}/git/db'
CARGO_CRATES = f'{CARGO_HOME}/vendor'
VENDORED_SOURCES = 'vendored-sources'


def rust_digest(b):
# The 0xff suffix matches Rust's behaviour
# https://doc.rust-lang.org/src/core/hash/mod.rs.html#611-616
digest = siphasher(b.encode() + b'\xff').decode('ascii').lower()
logging.debug("Hashing %r to %r", b, digest)
logging.debug('Hashing %r to %r', b, digest)
return digest


def canonical_url(url):
"Converts a string to a Cargo Canonical URL, as per https://github.com/rust-lang/cargo/blob/35c55a93200c84a4de4627f1770f76a8ad268a39/src/cargo/util/canonical_url.rs#L19"
logging.debug("canonicalising %s", url)
'Converts a string to a Cargo Canonical URL, as per https://github.com/rust-lang/cargo/blob/35c55a93200c84a4de4627f1770f76a8ad268a39/src/cargo/util/canonical_url.rs#L19'
logging.debug('canonicalising %s', url)
# Hrm. The upstream cargo does not replace those URLs, but if we don't then it doesn't work too well :(
url = url.replace("git+https://", "https://")
url = url.replace('git+https://', 'https://')
u = urlparse(url)
# It seems cargo drops query and fragment
u = ParseResult(u.scheme, u.netloc, u.path, None, None, None)
u = u._replace(path = u.path.rstrip('/'))

if u.netloc == "github.com":
u = u._replace(scheme = "https")
if u.netloc == 'github.com':
u = u._replace(scheme = 'https')
u = u._replace(path = u.path.lower())

if u.path.endswith(".git"):
u = u._replace(path = u.path[:-len(".git")])
if u.path.endswith('.git'):
u = u._replace(path = u.path[:-len('.git')])

return u

Expand All @@ -62,89 +62,101 @@ def load_cargo_lock(lockfile='Cargo.lock'):
cargo_lock = toml.load(f)
return cargo_lock

def get_git_sources(package):
name = package['name']
source = package['source']
revision = urlparse(source).fragment
branches = parse_qs(urlparse(source).query).get('branch', [])
if branches:
assert len(branches) == 1, f'Expected exactly one branch, got {branches}'
branch = branches[0]
else:
branch = 'master'

assert revision, 'The commit needs to be indicated in the fragement part'
canonical = canonical_url(source)
digest = rust_digest(canonical.geturl())
cargo_vendored_entry = {
canonical.geturl(): {
'git': canonical.geturl(),
'branch': branch,
#XXX 'rev': revision,
'replace-with': VENDORED_SOURCES,
}
}
git_sources = [
{
'type': 'git',
'url': canonical.geturl(),
'commit': revision,
'dest': f'{CARGO_CRATES}/{name}',
},
{
'type': 'shell',
'commands': [
f'git clone --bare {CARGO_CRATES}/{name} {CARGO_GIT_DB}/{name}-{digest}'
]
},
{
'type': 'shell',
'commands': [
# FIXME: This is an ugly workaround for imap-proto, https://github.com/djc/tokio-imap, which has workspaces in Cargo.toml
# The correct solution is to parse Cargo.toml.
# Then, however, we get very close to implementation details s.t. it seems smarter to patch cargo instead of
# reverse engineering its behaviour.
f'if test -d {CARGO_CRATES}/{name}/{name}; then '
f'mv {CARGO_CRATES}/{name} {CARGO_CRATES}/{name}.bak; '
f'cp -ar --dereference --reflink=auto {CARGO_CRATES}/{name}.bak/{name} {CARGO_CRATES}/{name}; '
f'rm -r {CARGO_CRATES}/{name}.bak; '
'fi',
],
},
{
'type': 'file',
# FIXME: Vendor is hard coded
'url': 'data:' + urlquote(json.dumps({'package': None, 'files': {}})),
'dest': f'{CARGO_CRATES}/{name}', #-{version}',
'dest-filename': '.cargo-checksum.json',
},
{
'type': 'shell',
'commands': [
f'echo rm -r {CARGO_CRATES}/{name}/.git',
# FIXME: Cargo does not copy .git/ and some other files
],
},
]
return (git_sources, cargo_vendored_entry)

def generate_sources(cargo_lock):
sources = []
cargo_git_sources = []
metadata = cargo_lock['metadata']
cargo_vendored_sources = {
VENDORED_SOURCES: {'directory': f'{CARGO_CRATES}'},
'crates-io': {'replace-with': VENDORED_SOURCES},
}
metadata = cargo_lock.get('metadata')
for package in cargo_lock['package']:
name = package['name']
version = package['version']
if 'source' in package:
source = package['source']
if source.startswith("git+"):
revision = urlparse(source).fragment
branches = parse_qs(urlparse(source).query).get("branch", [])
if branches:
assert len(branches) == 1, f"Expected exactly one branch, got {branches}"
branch = branches[0]
else:
branch = "master"

assert revision, "The commit needs to be indicated in the fragement part"
canonical = canonical_url(source)
reponame = canonical.path.rsplit('/', 1)[1]
hash = rust_digest(canonical.geturl())
shortcommit = revision[:8]
cargo_git_source = {
"canonical": canonical.geturl(),
"branch": branch,
"rev": revision,
}
git_sources = [
{
"type": "git",
"url": canonical.geturl(),
"commit": revision,
"dest": f'{CARGO_CRATES}/{name}',
},
{
"type": "shell",
"commands": [
f"git clone --bare {CARGO_CRATES}/{name} {CARGO_GIT_DB}/{name}-{hash}"
]
},
{
"type": "shell",
"commands": [
# FIXME: This is an ugly workaround for imap-proto, https://github.com/djc/tokio-imap, which has workspaces in Cargo.toml
# The correct solution is to parse Cargo.toml.
# Then, however, we get very close to implementation details s.t. it seems smarter to patch cargo instead of
# reverse engineering its behaviour.
f"if test -d {CARGO_CRATES}/{name}/{name}; then "
f"mv {CARGO_CRATES}/{name} {CARGO_CRATES}/{name}.bak; "
f"cp -ar --dereference --reflink=auto {CARGO_CRATES}/{name}.bak/{name} {CARGO_CRATES}/{name}; "
f"rm -r {CARGO_CRATES}/{name}.bak; "
"fi",
],
},
{
'type': 'file',
# FIXME: Vendor is hard coded
'url': "data:" + urlquote(open(("vendor/" + f"{name}/.cargo-checksum.json"), 'r').read()),
'dest': f'{CARGO_CRATES}/{name}', #-{version}',
'dest-filename': '.cargo-checksum.json',
},
{
"type": "shell",
"commands": [
f"echo rm -r {CARGO_CRATES}/{name}/.git",
# FIXME: Cargo does not copy .git/ and some other files
],
},
]
if source.startswith('git+'):
git_sources, cargo_vendored_entry = get_git_sources(package)
sources += git_sources
cargo_git_sources.append(cargo_git_source)
cargo_vendored_sources.update(cargo_vendored_entry)
continue

else:
key = f'checksum {name} {version} ({source})'
if key not in metadata:
print(f'{key} ({source}) not in metadata', file=sys.stderr)
if metadata is not None and key in metadata:
checksum = metadata[key]
elif 'checksum' in package:
checksum = package['checksum']
else:
logging.warning(f'{name} doesn\'t have checksum')
continue
checksum = metadata[key]
else:
print(f'{name} has no source', file=sys.stderr)
logging.debug(f"Package for {name}: {package}")
logging.warning(f'{name} has no source')
logging.debug(f'Package for {name}: {package}')
continue
sources += [
{
Expand All @@ -168,29 +180,12 @@ def generate_sources(cargo_lock):
'for c in *.crate; do tar -xf $c; done'
]
})
cargo_sources = {
'crates-io': {'replace-with': 'vendored-sources'},
'vendored-sources': {'directory': f'{CARGO_CRATES}'},
}
for cargo_git_source in cargo_git_sources:
# FIXME: Make those a proper attrib
canonical = cargo_git_source["canonical"]
branch = cargo_git_source["branch"]
revision = cargo_git_source["rev"]

key = canonical
value = {
"git": canonical,
"branch": branch,
# "rev": revision,
"replace-with": "vendored-sources",
}
cargo_sources[key] = value

logging.debug(f'Vendored sources: {cargo_vendored_sources}')
sources.append({
'type': 'file',
'url': 'data:' + urlquote(toml.dumps({
'source': cargo_sources,
'source': cargo_vendored_sources,
})),
'dest': CARGO_HOME,
'dest-filename': 'config'
Expand Down

0 comments on commit dd49b1a

Please sign in to comment.