From 82a4a1625ef5a82dd52dcedb5f9a9129560b6a55 Mon Sep 17 00:00:00 2001 From: Michael Baentsch Date: Fri, 6 Aug 2021 16:34:26 +0200 Subject: [PATCH] integrated src and doc upstream copy [skip ci] --- docs/algorithms/kem/kyber.yml | 24 +-- docs/algorithms/sig/dilithium.yml | 12 +- .../copy_from_upstream/copy_from_upstream.py | 3 +- .../update_pqclean_alg_docs.py | 190 ------------------ .../update_upstream_alg_docs.py | 73 ++++--- 5 files changed, 61 insertions(+), 241 deletions(-) delete mode 100755 scripts/copy_from_upstream/update_pqclean_alg_docs.py diff --git a/docs/algorithms/kem/kyber.yml b/docs/algorithms/kem/kyber.yml index c08c04d99f..bb2d4adf34 100644 --- a/docs/algorithms/kem/kyber.yml +++ b/docs/algorithms/kem/kyber.yml @@ -44,8 +44,8 @@ parameter-sets: - Darwin required_flags: - popcnt - - avx2 - bmi2 + - avx2 common-crypto: - SHA3: liboqs no-secret-dependent-branching-claimed: true @@ -75,12 +75,12 @@ parameter-sets: - Linux - Darwin required_flags: - - popcnt - - sse2 - - avx2 - aes + - avx2 + - popcnt - ssse3 - bmi2 + - sse2 common-crypto: - AES: pqcrystals-kyber_common_aes - SHA3: liboqs @@ -111,8 +111,8 @@ parameter-sets: - Darwin required_flags: - popcnt - - avx2 - bmi2 + - avx2 common-crypto: - SHA3: liboqs no-secret-dependent-branching-claimed: true @@ -142,12 +142,12 @@ parameter-sets: - Linux - Darwin required_flags: - - popcnt - - sse2 - - avx2 - aes + - avx2 + - popcnt - ssse3 - bmi2 + - sse2 common-crypto: - AES: pqcrystals-kyber_common_aes - SHA3: liboqs @@ -178,8 +178,8 @@ parameter-sets: - Darwin required_flags: - popcnt - - avx2 - bmi2 + - avx2 common-crypto: - SHA3: liboqs no-secret-dependent-branching-claimed: true @@ -209,12 +209,12 @@ parameter-sets: - Linux - Darwin required_flags: - - popcnt - - sse2 - - avx2 - aes + - avx2 + - popcnt - ssse3 - bmi2 + - sse2 common-crypto: - AES: pqcrystals-kyber_common_aes - SHA3: liboqs diff --git a/docs/algorithms/sig/dilithium.yml b/docs/algorithms/sig/dilithium.yml index 15471e1f2e..968e746cb1 100644 --- a/docs/algorithms/sig/dilithium.yml +++ b/docs/algorithms/sig/dilithium.yml @@ -133,10 +133,10 @@ parameter-sets: - Darwin - Linux required_flags: + - aes + - avx2 - popcnt - sse2 - - avx2 - - aes - ssse3 common-crypto: - AES: pqcrystals @@ -167,10 +167,10 @@ parameter-sets: - Darwin - Linux required_flags: + - aes + - avx2 - popcnt - sse2 - - avx2 - - aes - ssse3 common-crypto: - AES: pqcrystals @@ -201,10 +201,10 @@ parameter-sets: - Darwin - Linux required_flags: + - aes + - avx2 - popcnt - sse2 - - avx2 - - aes - ssse3 common-crypto: - AES: pqcrystals diff --git a/scripts/copy_from_upstream/copy_from_upstream.py b/scripts/copy_from_upstream/copy_from_upstream.py index 06bf886edd..0c466bc00f 100755 --- a/scripts/copy_from_upstream/copy_from_upstream.py +++ b/scripts/copy_from_upstream/copy_from_upstream.py @@ -13,6 +13,7 @@ from pathlib import Path import sys import json +import update_upstream_alg_docs # kats of all algs kats = {} @@ -512,7 +513,7 @@ def copy_from_upstream(): if not keepdata: shutil.rmtree('repos') - print("Remember to update the docs by running update_pqclean_alg_docs.py") + update_upstream_alg_docs.do_it(os.environ['LIBOQS_DIR']) def verify_from_upstream(): diff --git a/scripts/copy_from_upstream/update_pqclean_alg_docs.py b/scripts/copy_from_upstream/update_pqclean_alg_docs.py deleted file mode 100755 index dfffff14ee..0000000000 --- a/scripts/copy_from_upstream/update_pqclean_alg_docs.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: MIT - -import argparse -import os -import subprocess -import yaml -import inspect - -parser = argparse.ArgumentParser() -parser.add_argument("--liboqs-root", default=os.path.join("..", "..")) -parser.add_argument("-w", "--write-changes", dest="write_changes", action='store_true') -parser.add_argument("-v", "--verbosity", type=int) -args = parser.parse_args() - -if args.verbosity: - DEBUG = args.verbosity -else: - DEBUG = 0 - -if not args.write_changes: - print("--write-changes not set; changes will not be written out.") - -def shell(command, expect=0): - subprocess_stdout = None if DEBUG > 0 else subprocess.DEVNULL - ret = subprocess.run(command, stdout=subprocess_stdout, stderr=subprocess_stdout) - if ret.returncode != expect: - raise Exception("'{}' failed with error {}. Expected {}.".format(" ".join(command), ret, expect)) - -def load_yaml(filename, encoding='utf-8'): - with open(filename, mode='r', encoding=encoding) as fh: - return yaml.safe_load(fh.read()) - -def store_yaml(filename, contents, encoding='utf-8'): - with open(filename, mode='w', encoding=encoding) as fh: - yaml.dump(contents, fh, sort_keys=False, allow_unicode=True) - -def fetch_upstream(upstream_info): - work_dir_root = os.path.join(args.liboqs_root, 'scripts', 'copy_from_upstream', 'repos') - os.makedirs(work_dir_root, exist_ok=True) - - work_dir = os.path.join(work_dir_root, upstream_info['name']) - work_dotgit = os.path.join(work_dir, '.git') - if not os.path.exists(work_dotgit): - shell(['git', 'init', work_dir]) - shell(['git', '--git-dir', work_dotgit, 'remote', 'add', 'origin', upstream_info['git_url']]) - shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'remote', 'set-url', 'origin', upstream_info['git_url']]) - shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'fetch', '--depth=1', 'origin', upstream_info['git_commit']]) - shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'reset', '--hard', upstream_info['git_commit']]) - - return work_dir - -def rhs_if_not_equal(lhs, rhs, not_equal_msg): - if lhs != rhs: - if DEBUG > 0: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - print("Line {}: Discrepancy in {}: lhs: {}, rhs: {}".format(caller.lineno, not_equal_msg, lhs, rhs)) - return rhs - return lhs - -def update_pqclean_kem_alg_docs(kems, upstream_info, pqclean_root, meta_yaml_path_template): - for kem in kems: - if kem['upstream_location'] == 'pqclean': - oqs_yaml_path = os.path.join(args.liboqs_root, 'docs', 'algorithms', 'kem', '{}.yml'.format(kem['name'])) - if os.path.isfile(oqs_yaml_path): - oqs_yaml = load_yaml(oqs_yaml_path) - else: - continue - - # We assume that the ordering of "parameter-sets" - # in the OQS YAML files matches that of copy_from_upstream.yml - for index, scheme in enumerate(kem['schemes']): - if DEBUG > 0: - print("Examining {}'s META.yml.".format(scheme['pretty_name_full'])) - pqclean_meta_path = os.path.join(pqclean_root, meta_yaml_path_template.format_map(scheme)) - pqclean_yaml = load_yaml(pqclean_meta_path) - - oqs_yaml['type'] = rhs_if_not_equal(oqs_yaml['type'], pqclean_yaml['type'], "type") - oqs_yaml['principal-submitters'] = rhs_if_not_equal(oqs_yaml['principal-submitters'], pqclean_yaml['principal-submitters'], "principal-submitters") - - upstream_base_url = upstream_info['git_url'][:-len(".git")] - oqs_yaml['upstream'] = rhs_if_not_equal(oqs_yaml['upstream'], "{}/commit/{}".format(upstream_base_url, upstream_info['git_commit']), "upstream") - - if 'auxiliary-submitters' in pqclean_yaml: - oqs_yaml['auxiliary-submitters'] = rhs_if_not_equal(oqs_yaml['auxiliary-submitters'] if 'auxiliary-submitters' in oqs_yaml else '', pqclean_yaml['auxiliary-submitters'], "auxiliary-submitters") - - oqs_scheme_yaml = oqs_yaml['parameter-sets'][index] - - # TODO: PQClean and liboqs pretty-naming conventions for the - # following algorithms are out of sync. - if kem['name'] == 'classic_mceliece' or kem['name'] == 'hqc' or kem['name'] == 'ntru' or kem['name'] == 'saber': - oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], scheme['pretty_name_full'], "scheme pretty name") - else: - oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], pqclean_yaml['name'], "scheme pretty name") - - oqs_scheme_yaml['claimed-nist-level'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-nist-level'], pqclean_yaml['claimed-nist-level'], "claimed-nist-level") - oqs_scheme_yaml['claimed-security'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-security'], pqclean_yaml['claimed-security'], "claimed-security") - oqs_scheme_yaml['length-public-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-public-key'], pqclean_yaml['length-public-key'], "length-public-key") - oqs_scheme_yaml['length-ciphertext'] = rhs_if_not_equal(oqs_scheme_yaml['length-ciphertext'], pqclean_yaml['length-ciphertext'], "length-ciphertext") - oqs_scheme_yaml['length-secret-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-secret-key'], pqclean_yaml['length-secret-key'], "legnth-secret-key") - oqs_scheme_yaml['length-shared-secret'] = rhs_if_not_equal(oqs_scheme_yaml['length-shared-secret'], pqclean_yaml['length-shared-secret'], "length-shared-secret") - - for impl_index, impl in enumerate(oqs_scheme_yaml['implementations']): - for pqclean_impl in pqclean_yaml['implementations']: - if impl['upstream-id'] == pqclean_impl['name']: - break - - if 'supported_platforms' in pqclean_impl: - impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], pqclean_impl['supported_platforms'], "supported-platforms") - else: - impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], "all", "supported-platforms") - - oqs_scheme_yaml['implementations'][impl_index] = impl - - oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml - - if args.write_changes: - store_yaml(oqs_yaml_path, oqs_yaml) - -def update_pqclean_sig_alg_docs(sigs, upstream_info, pqclean_root, meta_yaml_path_template): - for sig in sigs: - if sig['upstream_location'] == 'pqclean': - oqs_yaml_path = os.path.join(args.liboqs_root, 'docs', 'algorithms', 'sig', '{}.yml'.format(sig['name'])) - if os.path.isfile(oqs_yaml_path): - oqs_yaml = load_yaml(oqs_yaml_path) - else: - continue - - # We assume that the ordering of "parameter-sets" - # in the OQS YAML files matches that of copy_from_upstream.yml - for index, scheme in enumerate(sig['schemes']): - if DEBUG > 0: - print("Examining {}'s META.yml.".format(scheme['pretty_name_full'])) - pqclean_meta_path = os.path.join(pqclean_root, meta_yaml_path_template.format_map(scheme)) - pqclean_yaml = load_yaml(pqclean_meta_path) - - oqs_yaml['type'] = rhs_if_not_equal(oqs_yaml['type'], pqclean_yaml['type'], "type") - oqs_yaml['principal-submitters'] = rhs_if_not_equal(oqs_yaml['principal-submitters'], pqclean_yaml['principal-submitters'], "principal-submitters") - - upstream_base_url = upstream_info['git_url'][:-len(".git")] - oqs_yaml['upstream'] = rhs_if_not_equal(oqs_yaml['upstream'], "{}/commit/{}".format(upstream_base_url, upstream_info['git_commit']), "upstream") - - if 'auxiliary-submitters' in pqclean_yaml: - oqs_yaml['auxiliary-submitters'] = rhs_if_not_equal(oqs_yaml['auxiliary-submitters'] if 'auxiliary-submitters' in oqs_yaml else '', pqclean_yaml['auxiliary-submitters'], "auxiliary-submitters") - - oqs_scheme_yaml = oqs_yaml['parameter-sets'][index] - - # TODO: PQClean and liboqs pretty-naming conventions for the - # following algorithms are out of sync. - if sig['name'] == 'sphincs' or sig['name'] == 'rainbow': - oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], scheme['pretty_name_full'], "scheme pretty name") - else: - oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], pqclean_yaml['name'], "scheme pretty name") - - oqs_scheme_yaml['claimed-nist-level'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-nist-level'], pqclean_yaml['claimed-nist-level'], "claimed-nist-level") - oqs_scheme_yaml['claimed-security'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-security'], 'EUF-CMA', "claimed-security") - oqs_scheme_yaml['length-public-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-public-key'], pqclean_yaml['length-public-key'], "length-public-key") - oqs_scheme_yaml['length-secret-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-secret-key'], pqclean_yaml['length-secret-key'], "legnth-secret-key") - oqs_scheme_yaml['length-signature'] = rhs_if_not_equal(oqs_scheme_yaml['length-signature'], pqclean_yaml['length-signature'], "length-signature") - - for impl_index, impl in enumerate(oqs_scheme_yaml['implementations']): - for pqclean_impl in pqclean_yaml['implementations']: - if impl['upstream-id'] == pqclean_impl['name']: - break - - if 'supported_platforms' in pqclean_impl: - impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], pqclean_impl['supported_platforms'], "supported-platforms") - else: - impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], "all", "supported-platforms") - - oqs_scheme_yaml['implementations'][impl_index] = impl - - oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml - - if args.write_changes: - store_yaml(oqs_yaml_path, oqs_yaml) - -instructions = load_yaml( - os.path.join(args.liboqs_root, 'scripts', 'copy_from_upstream', 'copy_from_upstream.yml'), - encoding='utf-8') - -for upstream in instructions['upstreams']: - if upstream['name'] == 'pqclean': - pqclean_info = upstream - break -pqclean_root = fetch_upstream(pqclean_info) - -update_pqclean_kem_alg_docs(instructions['kems'], pqclean_info, pqclean_root, pqclean_info['kem_meta_path']) -update_pqclean_sig_alg_docs(instructions['sigs'], pqclean_info, pqclean_root, pqclean_info['sig_meta_path']) diff --git a/scripts/copy_from_upstream/update_upstream_alg_docs.py b/scripts/copy_from_upstream/update_upstream_alg_docs.py index ab549db01a..e28f49386e 100755 --- a/scripts/copy_from_upstream/update_upstream_alg_docs.py +++ b/scripts/copy_from_upstream/update_upstream_alg_docs.py @@ -8,19 +8,7 @@ import yaml import inspect -parser = argparse.ArgumentParser() -parser.add_argument("--liboqs-root", default=os.path.join("..", "..")) -parser.add_argument("-w", "--write-changes", dest="write_changes", action='store_true') -parser.add_argument("-v", "--verbosity", type=int) -args = parser.parse_args() - -if args.verbosity: - DEBUG = args.verbosity -else: - DEBUG = 0 - -if not args.write_changes: - print("--write-changes not set; changes will not be written out.") +DEBUG = 0 def shell(command, expect=0): subprocess_stdout = None if DEBUG > 0 else subprocess.DEVNULL @@ -36,8 +24,8 @@ def store_yaml(filename, contents, encoding='utf-8'): with open(filename, mode='w', encoding=encoding) as fh: yaml.dump(contents, fh, sort_keys=False, allow_unicode=True) -def fetch_upstream(upstream_info): - work_dir_root = os.path.join(args.liboqs_root, 'scripts', 'copy_from_upstream', 'repos') +def fetch_upstream(liboqs_root, upstream_info): + work_dir_root = os.path.join(liboqs_root, 'scripts', 'copy_from_upstream', 'repos') os.makedirs(work_dir_root, exist_ok=True) work_dir = os.path.join(work_dir_root, upstream_info['name']) @@ -79,12 +67,12 @@ def get_oqs_yaml(param_list, name): print(param_list) exit(1) -# Merge documentation contained in args.liboqs_root/docs/algorithms/kem/kem['name'].yml with upstream information: +# Merge documentation contained in liboqs_root/docs/algorithms/kem/kem['name'].yml with upstream information: # Args: # kems: List of kems in copy_from_upstream.yml # upstream_info: Hashtable of upstream information (keyed by upstream source) # incl. entry: 'upstream_root' pointing to local folder containing source code -def update_upstream_kem_alg_docs(kems, upstream_info): +def update_upstream_kem_alg_docs(liboqs_root, kems, upstream_info, write_changes=False): for kem in kems: ui = get_upstream_info(upstream_info, kem['upstream_location']) upstream_root = ui['upstream_root'] @@ -92,7 +80,7 @@ def update_upstream_kem_alg_docs(kems, upstream_info): if DEBUG > 1: print("Working on KEM %s using path %s and META file %s" % (kem, upstream_root, meta_yaml_path_template)) if True: # for all upstream sources: - oqs_yaml_path = os.path.join(args.liboqs_root, 'docs', 'algorithms', 'kem', '{}.yml'.format(kem['name'])) + oqs_yaml_path = os.path.join(liboqs_root, 'docs', 'algorithms', 'kem', '{}.yml'.format(kem['name'])) if os.path.isfile(oqs_yaml_path): oqs_yaml = load_yaml(oqs_yaml_path) else: @@ -157,10 +145,10 @@ def update_upstream_kem_alg_docs(kems, upstream_info): oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml - if args.write_changes: + if write_changes: store_yaml(oqs_yaml_path, oqs_yaml) -def update_upstream_sig_alg_docs(sigs, upstream_info): +def update_upstream_sig_alg_docs(liboqs_root, sigs, upstream_info, write_changes=False): for sig in sigs: ui = get_upstream_info(upstream_info, sig['upstream_location']) upstream_root = ui['upstream_root'] @@ -168,7 +156,7 @@ def update_upstream_sig_alg_docs(sigs, upstream_info): if DEBUG > 1: print("Working on SIG %s using path %s and META file %s" % (sig, upstream_root, meta_yaml_path_template)) if True: # for all upstream sources: - oqs_yaml_path = os.path.join(args.liboqs_root, 'docs', 'algorithms', 'sig', '{}.yml'.format(sig['name'])) + oqs_yaml_path = os.path.join(liboqs_root, 'docs', 'algorithms', 'sig', '{}.yml'.format(sig['name'])) if os.path.isfile(oqs_yaml_path): oqs_yaml = load_yaml(oqs_yaml_path) else: @@ -231,16 +219,37 @@ def update_upstream_sig_alg_docs(sigs, upstream_info): oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml - if args.write_changes: + if write_changes: store_yaml(oqs_yaml_path, oqs_yaml) -instructions = load_yaml( - os.path.join(args.liboqs_root, 'scripts', 'copy_from_upstream', 'copy_from_upstream.yml'), - encoding='utf-8') - -for upstream in instructions['upstreams']: - if 'git_url' in upstream.keys(): - upstream['upstream_root'] = fetch_upstream(upstream) - -update_upstream_kem_alg_docs(instructions['kems'], instructions['upstreams']) -update_upstream_sig_alg_docs(instructions['sigs'], instructions['upstreams']) +def do_it(liboqs_root): + if liboqs_root == None: + parser = argparse.ArgumentParser() + parser.add_argument("--liboqs-root", default=os.path.join("..", "..")) + parser.add_argument("-w", "--write-changes", dest="write_changes", action='store_true') + parser.add_argument("-v", "--verbosity", type=int) + args = parser.parse_args() + + if args.verbosity: + DEBUG = args.verbosity + + liboqs_root = args.liboqs_root + write_changes = args.write_changes + else: + write_changes = True + + if not write_changes: + print("--write-changes not set; changes will not be written out.") + instructions = load_yaml( + os.path.join(liboqs_root, 'scripts', 'copy_from_upstream', 'copy_from_upstream.yml'), + encoding='utf-8') + + for upstream in instructions['upstreams']: + if 'git_url' in upstream.keys(): + upstream['upstream_root'] = fetch_upstream(liboqs_root, upstream) + + update_upstream_kem_alg_docs(liboqs_root, instructions['kems'], instructions['upstreams'], write_changes) + update_upstream_sig_alg_docs(liboqs_root, instructions['sigs'], instructions['upstreams'], write_changes) + +if __name__ == "__main__": + do_it(None)