Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build repository and file details for atomic repos. #7

Merged
merged 2 commits into from
Dec 6, 2014
Merged

Build repository and file details for atomic repos. #7

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
83ca21b
Build repository and file details for atomic repos.
ralphbean Dec 6, 2014
3a70913
Typofix.
ralphbean Dec 6, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 89 additions & 32 deletions utility/update-master-directory-list
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,10 @@ If the directory does not contain too many files, it will register them all
and thus will check them all on the mirrors.
The threshold is stored in: `short_filelist` and is currently at 10.

If the script finds a yum repository (finds a repo data), it will create a
repository object (cf `make_repository`) which is basically a mapping
between a yum repo name (ie: Fedora-20-updates) and a directory
(/pub/fedora/linux/updates/....)

If the script finds a yum or atomic repository (finds a repo data or an atomic
summary file), it will create a repository object (cf `make_repository`) which
is basically a mapping between a yum repo name (ie: Fedora-20-updates) and a
directory (/pub/fedora/linux/updates/....)

TODO: test IRL
"""
Expand Down Expand Up @@ -236,35 +235,56 @@ def make_file_details_from_checksums(session, diskpath, relativeDName, D):
timestamp=ctime,
size=size)
session.add(fd)
session.commit(fd)
session.commit()


def make_repomd_file_details(session, diskpath, relativeDName, D, category):
if diskpath is None: return
if not relativeDName.endswith(u'/repodata'): return
absolutepath = os.path.join(diskpath, relativeDName, 'repomd.xml')
def make_repo_file_details(session, diskpath, relativeDName, D, category, target):

warning = "Won't make repo file details"

if diskpath is None:
logger.warning("%s: diskpath is None" % warning)
return

# For yum repos and ostree repos
allowed_targets = ['repomd.xml', 'summary']
if target not in allowed_targets:
logger.warning("%s: %r not in %r" % (warning, target, allowed_targets))
return

absolutepath = os.path.join(diskpath, relativeDName, target)

if not os.path.exists(absolutepath):
logger.warning("%s: %r does not exist" % (warning, absolutepath))
return

try:
f = open(absolutepath, 'r')
repomd = f.read()
contents = f.read()
f.close()
except:
return
size = len(repomd)
md5 = hashlib.md5(repomd).hexdigest()
sha1 = hashlib.sha1(repomd).hexdigest()
sha256 = hashlib.sha256(repomd).hexdigest()
sha512 = hashlib.sha512(repomd).hexdigest()

yumrepo = yum.repoMDObject.RepoMD('repoid', absolutepath)
if 'timestamp' not in yumrepo.__dict__:
set_repomd_timestamp(yumrepo)
timestamp = yumrepo.timestamp

size = len(contents)
md5 = hashlib.md5(contents).hexdigest()
sha1 = hashlib.sha1(contents).hexdigest()
sha256 = hashlib.sha256(contents).hexdigest()
sha512 = hashlib.sha512(contents).hexdigest()

if target == 'repomd.xml':
yumrepo = yum.repoMDObject.RepoMD('repoid', absolutepath)
if 'timestamp' not in yumrepo.__dict__:
set_repomd_timestamp(yumrepo)
timestamp = yumrepo.timestamp
elif target == 'summary':
# TODO -- ostree repos may have a timestamp in their summary file
# someday. for now, just use the system mtime.
timestamp = os.path.getmtime(absolutepath)

fd = mirrormanager2.lib.get_file_detail(
session,
directory_id=D.id,
filename='repomd.xml',
filename=target,
sha1=sha1,
md5=md5,
sha256=sha256,
Expand All @@ -274,15 +294,15 @@ def make_repomd_file_details(session, diskpath, relativeDName, D, category):
if not fd:
fd = FileDetail(
directory_id=D.id,
filename='repomd.xml',
filename=target,
sha1=sha1,
md5=md5,
sha256=sha256,
sha512=sha512,
timestamp=timestamp,
size=size)
session.add(fd)
session.commit(fd)
session.commit()


def move_repository_from_development(prefix, arch, newdir):
Expand All @@ -298,16 +318,39 @@ def move_repository_from_development(prefix, arch, newdir):
return repo


def make_repository(session, directory, relativeDName, category):
repo = None
(ver, arch) = guess_ver_arch_from_path(session, category, relativeDName)
if ver is None or arch is None:
return None
def make_repository(session, directory, relativeDName, category, target):

warning = "Won't make repository object"

# For yum repos and ostree repos
allowed_targets = ['repomd.xml', 'summary']
if target not in allowed_targets:
logger.warning("%s: %r not in %r" % (warning, target, allowed_targets))
return

if target == 'repomd.xml':
(ver, arch) = guess_ver_arch_from_path(session, category, relativeDName)
if ver is None or arch is None:
logger.warning("%s: could not guess version and arch %r, %r" % (
warning, ver, arch))
return None
elif target == 'summary':
# For ostree, we someday need to actually extract the arch information
# from the ostree repo, but for now (F21 and F22) we will only be
# shipping x86_64, so we hardcode that. At present, it is not possible
# to query an ostree repo for the arch information. Bug walters about
# this.
arch = mirrormanager2.lib.get_arch_by_name(session, 'x86_64')
# Furthermore, we'll grab the version piece from the path which looks
# like atomic/rawhide or atomic/21.
ver = relativeDName.rstrip('/').split('/')[-1]

# stop making duplicate Repository objects.
if len(directory.repositories) > 0:
logger.warning("%s: directory already has a repository")
return None

repo = None
prefix = repo_prefix(relativeDName, category, ver)
try:
# historically, Repository.name was a longer string with
Expand Down Expand Up @@ -395,6 +438,7 @@ def fill_category_directories_from_rsync(
category_directories[relativeDName] = {
'files':{},
'isRepository':False,
'isAtomic':False,
'readable':readable,
'ctime':ctime,
'changed':True}
Expand Down Expand Up @@ -513,9 +557,20 @@ def sync_category_directories(
for relativeDName, value in category_directories.iteritems():
d = category.directory_cache[relativeDName]
D = mirrormanager2.lib.get_directory_by_id(session, d.id)
if value['isRepository']:
make_repository(session, D, relativeDName, category)
make_repomd_file_details(session, diskpath, relativeDName, D, category)

target = None
if 'repomd.xml' in value['files']:
target = 'repomd.xml'
elif 'summary' in value['files']:
target = 'summary'
else:
continue

if value['isRepository'] or value['isAtomic']:
make_repository(session, D, relativeDName, category, target)

make_repo_file_details(
session, diskpath, relativeDName, D, category, target)

Directory.age_file_details(session, config)

Expand Down Expand Up @@ -616,6 +671,7 @@ def sync_directories_from_disk(
if not readable or parent_dir(relativeDName) in category.unreadable_dirs:
category.unreadable_dirs.add(relativeDName)
isRepo = u'repodata' in dirnames
isAtomic = u'summary' in filenames and u'objects' in dirnames

changed = (d_ctime != ctime)
if changed:
Expand All @@ -627,6 +683,7 @@ def sync_directories_from_disk(
category_directories[relativeDName] = {
'files':{},
'isRepository':isRepo,
'isAtomic':isAtomic,
'readable':readable,
'ctime':ctime,
'changed':changed}
Expand Down