diff --git a/changelogs/unreleased/5971-sseago b/changelogs/unreleased/5971-sseago new file mode 100644 index 0000000000..ed8e9986ef --- /dev/null +++ b/changelogs/unreleased/5971-sseago @@ -0,0 +1 @@ +Follow-on fixes for BIAv2 controller work diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 1cfa693145..823efd976c 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -420,19 +420,19 @@ spec: status: description: BackupStatus captures the current status of a Velero backup. properties: - asyncBackupItemOperationsAttempted: - description: AsyncBackupItemOperationsAttempted is the total number - of attempted async BackupItemAction operations for this backup. + backupItemOperationsAttempted: + description: BackupItemOperationsAttempted is the total number of + attempted async BackupItemAction operations for this backup. type: integer - asyncBackupItemOperationsCompleted: - description: AsyncBackupItemOperationsCompleted is the total number - of successfully completed async BackupItemAction operations for - this backup. + backupItemOperationsCompleted: + description: BackupItemOperationsCompleted is the total number of + successfully completed async BackupItemAction operations for this + backup. type: integer - asyncBackupItemOperationsFailed: - description: AsyncBackupItemOperationsFailed is the total number of - async BackupItemAction operations for this backup which ended with - an error. + backupItemOperationsFailed: + description: BackupItemOperationsFailed is the total number of async + BackupItemAction operations for this backup which ended with an + error. type: integer completionTimestamp: description: CompletionTimestamp records the time a backup was completed. @@ -476,8 +476,8 @@ spec: - InProgress - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed - - FinalizingAfterPluginOperations - - FinalizingAfterPluginOperationsPartiallyFailed + - Finalizing + - FinalizingPartiallyFailed - Completed - PartiallyFailed - Failed diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index e89df169e4..43eb169af2 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,7 +30,7 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WAo\xdc6\x13\xbd\xebW\f\xf2\x1dr\xf9\xa4M\xd0C\v\xddR\xb7\x05\x82&\x86a\a\xbe\x14=P\xe4\xec.c\x8ad\xc9\xe1\xa6ۢ\xff\xbd\x18R\xf2j%\xd9\x1b\a\xa8n\"\x87of\xde\xcc\x1bQU]ו\xf0\xfa\x1eC\xd4ζ \xbc\xc6?\t-\xbf\xc5\xe6\xe1\x87\xd8h\xb79\xbc\xad\x1e\xb4U-\\\xa5H\xae\xbf\xc5\xe8R\x90\xf8\x13n\xb5դ\x9d\xadz$\xa1\x04\x89\xb6\x02\x10\xd6:\x12\xbc\x1c\xf9\x15@:K\xc1\x19\x83\xa1ޡm\x1eR\x87]\xd2Fa\xc8\xe0\xa3\xebÛ\xe6\xfb\xe6M\x05 \x03\xe6\xe3\x9ft\x8f\x91D\xef[\xb0ɘ\n\xc0\x8a\x1e[\xe8\x84|H>\xa0wQ\x93\v\x1acs@\x83\xc15\xdaUѣd\xb7\xbb\xe0\x92o\xe1\xb4QN\x0f!\x95t~\xcc@\xb7#\xd01o\x19\x1d\xe9\xd7\xd5\xed\x0f:R6\xf1&\x05a\xd6\x02\xc9\xdbQ\xdb]2\",\f\xd8A\x94\xcec\v\xd7\x1c\x8b\x17\x12U\x050P\x90c\xabA(\x95I\x15\xe6&hK\x18\xae\x9cI\xfdHf\r\x9f\xa3\xb37\x82\xf6-4#\xed͂\xb2l;\x12\xf6n\x87\xc3;\x1dٹ\x12\x84K0f\xae9\xc5\xfa\xe9\xe8\xf1\f\xe5D\x04L\xf6\nb\xa4\xa0\xed\xae:\x19\x1f\xde\x16*\xe4\x1e{\xd1\x0e\xb6Σ}w\xf3\xfe\xfe\xbb\xbb\xb3e\x00\x1f\x9c\xc7@z,Oy&}9Y\x05P\x18eОr\u05fcf\xc0b\x05\x8a\x1b\x12#\xd0\x1eGNQ\r1\x80\xdb\x02\xedu\x84\x80>`D[Z\xf4\f\x18\xd8HXp\xddg\x94\xd4\xc0\x1d\x06\x86\x81\xb8w\xc9(\xee\xe3\x03\x06\x82\x80\xd2\xed\xac\xfe\xeb\x11;\x02\xb9\xec\xd4\b¡GNO\xae\xa1\x15\x06\x0e\xc2$\xfc?\b\xab\xa0\x17G\b\xc8^ \xd9\t^6\x89\r|t\x01AۭkaO\xe4c\xbb\xd9\xec4\x8dz\x94\xae\xef\x93\xd5t\xdcdi\xe9.\x91\vq\xa3\xf0\x80f\x13\xf5\xae\x16A\xee5\xa1\xa4\x14p#\xbc\xaes\xe86k\xb2\xe9\xd5\xff\u00a0\xe0\xf8\xfa,\xd6E-˓\xc5\xf2L\x05X-\xa0#\x88\xe1h\xc9\xe2D4/1;\xb7?\xdf}\x82\xd1u.Ɯ\xfd\xcc\xfb\xe9`<\x95\x80\t\xd3v\x8b\xa1\x14q\x1b\\\x9f1\xd1*ﴥ\xfc\"\x8dF;\xa7?\xa6\xae\xd7\xc4u\xff#a$\xaeU\x03WyHA\x87\x90<\xabA5\xf0\xde\u0095\xe8\xd1\\\x89\x88\xffy\x01\x98\xe9X3\xb1_W\x82\xe9|\x9d\x1b\x17\xd6&\x1b\xe3\b|\xa2^\xf3\xb1v\xe7Qr\xf9\x98A>\xaa\xb7Zfm\xc0\xd6\x05\x10\v\xfb\xe6\fz]\xba\xfc\x94\xe1wG.\x88\x1d~p\x05sn\xb4\x1a\xdb\xec\xcc\x18\x1cO\x96\"c\\7\\`\x03\xd0^\xd0D\xbf$\xb4}\x1c\x03\xab\xf90\x92-S\bhi\x80\xc97\x88o\x1e\x99FD\x9a\x8c\v\xbe\xcd]\xe8\x80\x0f\xcb\x13c`\f\x06\xc4\v\xd3\xf9\xf2E̿\xba\xb9hk\x93e\xebB/\xa8\\\x17k\x06ZX\xf0\xb5\\t\x06[\xa0\x90\x96\xdb\xcf\xcdQ\x8cQ\xec.e\xf7\xb1X\x95\xcb\xc5p\x04D\xe7\x12=A=\xed\x97Q\xc0\x85r\\\x88\xd4\xefE\xbc\x14\xe7\r۬5\xc4\xec{\xf5\\\bO\xcd\xcck\xfc\xb2\xb2z\x8bB-u\\õ\xa3\xf5\xad'3\\U\xc5b1\xf2=LM\xea\x1c\x8b\x90\xa7+\xa9{\xbcW\xb6\xf0\xf7?\xd5IXBJ\xf4\x84\xeaz\xfe\a6\xcc\xf7\xf1\x87*\xbfJg\xcb\x0fPl\xe1\xb7߫\xe2\n\xd5\xfd\xf8\x93ċ\xff\x06\x00\x00\xff\xff\xc8p\x98۸\x0e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c9n\xef\xfa\x15(\xe5\xc1wU\x9aѺ\xf2\x90\x94\u07bc\xb2]Q\xed\xc6VY:\xdfC\x92\aN7f\x86+6\xd9G\xb2G\x9eK忧\b\xb2\xbf\xd9ݜ\xb1t\xb7we\xbe캇\x04A\x00\x04\x01\x10\x84.V\xab\xd5\x05+\xf9WԆ+y\x03\xac\xe4\xf8͢t\xff2\xeb\xa7\u007f7k\xae\xae\x0fo/\x9e\xb8\xcco\xe0\xb62V\x15_ШJg\xf8\x1e\xb7\\r˕\xbc(в\x9cYvs\x01\xc0\xa4T\x96\xb9\xcf\xc6\xfd\x13 S\xd2j%\x04\xea\xd5\x0e\xe5\xfa\xa9\xda\xe0\xa6\xe2\"GM\xc0\xeb\xa9\x0f?\xad\xffm\xfd\xd3\x05@\xa6\x91\x86?\xf2\x02\x8deEy\x03\xb2\x12\xe2\x02@\xb2\x02o`ò\xa7\xaa4\xeb\x03\n\xd4j\xcdՅ)1ss\xed\xb4\xaa\xca\x1bh\u007f\xf0C\x02\x1e~\r?\xd3h\xfa \xb8\xb1\xbft>\xfeʍ\xa5\x1fJQi&\x9a\x99\xe8\x9b\xe1rW\t\xa6\xeb\xaf\x17\x00&S%\xde\xc0'7E\xc92\xcc/\x00\xc2rh\xcaU@\xf8\xf0\xd6C\xc8\xf6X0\x8f\v\x80*Q\xbe\xbb\xbf\xfb\xfa\xaf\x0f\xbd\xcf\x009\x9aL\xf3\xd2\x12Qѽ#\x90-\v\x1c\xc1\xb8ܢ\xf6L\xdcjU\x10L\x94\xb9\x97>\x12]\xc1Q\x0e\xc9o\xaaM\xc1\xad\xe3\xfb_*4N\xc8\xd5\x1anI\xc5\xc0\x06\xa1*s'\x99k\xb8\x93p\xcb\n\x14\xb7\xcc\xe0\xab3\xc0Qڬ\x1ca\xd3X\xd0Վ\xc3Ξj\x9d\x1fj]6\xc1/\xaf\x10\x1eJ\xccz\x1bƍ\xe2[\x9eѶ\x80\xadҭ\xbe\xf0\xeaj\xdd\x03\x19߲\xaee\x86?HV\x9a\xbd\xb2N\xff\xaa\xca\x0e{\f\x10\xba}\xb8\x1b\f\xa8\x91\t\xa8\x91Z\xa9\f\xe6n\x9f=3n\x1dz#\x98\xe0\x00\xc1W\xd205<\xd24\x95\x01[iI\xbb\xf4\v\xb2\xfc\xf8\xa8\xfed\x10\U0008a135>+\xae`\x83[\xa51\x02W\xa3\x1b\xef:\xa3֎0\x86PR\x95]\xc3\xe3\x1e\x1d\x19Y%l\x90{n\xe0\xedOPpYY\\\x8f\xa0M0\xd8\x13\x85\xc0\xf8\x15\x98G\xf5\xd1xV-\x90\xef\xfdİ\x0e\x11\x9f\xf7h\xf7\xa8\xa1T\xb5\n\x8e\xacr\xcb\x05\x829\x1a\x8bE\xe0x\xad\xf86\x81\xfa$\x14B\x04\x10\x066\xc7\x1a\xe7\xf1:\xddy\xcb6\x02o\xc0\xeaj<\x9d'\xc3F)\x81l\xa8\x84\x87t\xf8\x82\xc6\xf2l\x81\n\x97C2\xf8Q\x11\"\xe8\xf0\x03\xad-B\x87M+k\x96=!\xb0\x9a\x1a\xeep\x10\xa2C\xc4\x1e\x05\xe0\xbf%\xbcw\x9a+s\xfad\x8c-\x04\xcd\xc5Q\x90\xb6\x94\n\x84\x92;\xd4~6w*)K_^\x85\x9c\x1e\xf13\x88\xe9\a\xd2\xf6\x92^m;:tc\xc8\t\xc2\xed\u06dd\xf7\xf0\x1a\xf6p\x03w\xd2\xf9-\x81\x1et#\u09db?\x1f\xfa\xad\xa8\f\x05\x89\xa5\x92+:*ױ\x99<\xb1\x13A*\xdd\xe3\xc8\x18\xb5fR?a\"\xd8Gw\x92\xf8\xf1\xfe\x8eC\xb0\f\xf3:\xc6I\x91yfq\xc73(P\xef\xe6\x0e\x8en+\x9d~OC!Q\xeb\xfav\xa2\x84\xa5\x1d\xedu\v\xaa;\x1a\x82뷕۹\t\xbdjf/v\x9d\b\xc8Ow]^\x11\x1d\xb1d\u007f,R\x97\xe59ݥ2q\u007f\x82\xc6?\x81\x17\xe3\xb3\xdf#\xe6OȂQ\x88\xf4\u007f\xdd1G\x02\xfd\u007fP2\xae\x13\xf6\xf0;\xba\x1a\x15\xd8\x1b\x1b\xa2X\xddi\xdc\f܀\xe3\uf049\xf1UOdq\xca\xe9\x16\x14\xfe Wۑ\xc5r\x05\xcf{e\xfc\x99J\xa1\xd9E\x90\xdc\xc0\xe5\x13\x1e/\xafFz\xe0\xf2N^\xfa\x03\xfedu\xd3X\vJ\x8a#\\\xd2\xd8\xcb\xef1\x82\x12%1\xa9\x1b]A\xa7\x9a\xcaΗ\xac-\x017\xb0\xb9wuf\xee\x1c\xd6IrX*\x13\xb9M\x9a@\xe5^\x19\xeb#\x8b=\xb3\xf4\x94(\x16x\x19\n\xd1+`[\u007f\xf3\xadt}\xa7\xe9\xd4\xde \xe0\xea\xb8f\xe65\xaccc\x13\x11\xf3@\x9dcu\xd9\xee`\xafO/\xfdE'M\xc222.\x16\xe1\x96Zeh̼\x88$h\xeb\x85 a\x13 dށ\xf1\x17\x86\xf3Aɺ\xa5\x1b\xa4\x8eH'\x9a\xf2\x1f\xbeu\xa2\x97n\xf3\xbb\u007f/\tߩx\x01\xed٢`Û\xf1$\x14o\xfd\xc8z\x9b\x04@\xde5л\x8a\xb6z\xba\x05\x19\x04\xe9\xf7pL\x17\\\xde\xd1\x04\xf0\xf6ŏ\xf5FI\xe29\x86\xfbm=\xb6%z\xf3\x81vo\xaaE\xa4(r\xaf\xb1ǹq\x9c\xdb\x19\x8a\x89 \xa5\xb2\xddp\x82\x83[\xaa\xfc\x8d\x81-\xd7\xc6v\x11M\x15\x8aja\xf7\xb7\xedT\xcfI~\xd0\xfa,\xc7\xe9\xb3\x1f\xd9\td\xed\xd5s\x9d_0y\x15\x1bkt)\x84\xc0\xb7\xc0-\xa0\xccT%)\xfc\xe2\xb6:M\xe1Y\xe0\x15t2\xc9\xd2\x14\x84k(\xab\"\x8d\x00+\x92:.g\xe34\xdd\xee\x1f\x19\x17\xaf\xc16;\x95\x86\x11k=\xb6\xd5\xf9\x18\xddD\x91\x82}\xe3EU\x00+\x1c\xe9Sݞ\xad\xcf\xe2\xe8q\xbc\xc9\xe5 \xb8t\x8cX\xe56U)Ц\xeeH\x9f\xb5ᶉ\xe196\as\x90\x02%\x81\xc1\x96q1qy>n'\xd1\xf6\x14_#(\x8b\x97s\"\xd2&_\x11)\x12\x02\xb1\x89\xc6⼶.u\xba\xa9x\xaf1\xcd<[\nJ\xd7\xe6Y\xa9\xb9\x93%\xf5\xd2\x16Z\x101&\x8f?L\xb4Q\xfba\xa2-\xb4\x1f&\xdad\xfba\xa2-\xb7\x1f&Zh?L\xb4\xba\xfd0\xd1~\x98hs\xdd\xe6\xb4\xf5\x12F\xfe\xc5\xc9ď\x8bX$\\Oϡ8\x03?dS\xdc\xfa\xd7'\xa9\x19\x96w\xf1Q\x91\xac\xe0\xf0\xaceE/rb\x12\xd0&]\xb4GI\x93r\xe96H-\xde>\x81~!\t\xf3;\xb2oS\x12~\x96\xd2|\xfay\xa6M\x9aM\x9dh\xaa\xeaI\"t\xa8_\xf68\xb3\xb7\x9bC\xd2\xcf\xd7!;\xb7\xc6\xf4\uf783\x9a\x90\x8a\xb3\x90\x803\x9f\x98;G\xaf\x81\xeb\xd1'\x98\xee%\x8c\xfe~\xe8e\xb1\xf8\\\x06QN{\xc4q\x17\x19\xb2\xf4\x8c#\xb2\x1e\xf2Q\xccQf{\xad\xa4\xaaLpk\x1c\xf4w\xe4]\x85\x1b\x17\xca6\x89\xbd\xbd\x88\xc0|\v{UE\x12Cgh\xb7\x90&4\x9d\x1c\x14\xae\xc2в\xc3\xdbu\xff\x17\xabB\xaa\x10\xecɊi\xde\xffJK&:;\x85\xa8\x9f\"4qF\x9dzg\x96\x9e)\x9d\x9e$4\x9f\xd5sJj\xd00\xf1g\x12\xe8rBP\x8a뼐\xfcsF\xcaOb\xba\xe7w\xdf\f\xa6$\xf5\x9c\x95ʳ\x98\x11\x99\x98\xc0\xd3O͙\ayB\xdaN\x12q\x96StNN\xcc\t\x890\xb3\xebHNlj$\xda\xcc\x02\x9eL\u0099K\xafY\bˍSoғjfAS\xc2\xcdr*\xcd\xcb%̾\x84\x130\xadj\x16\xd3a\x16\x9d\x84y\xfc\x16\x13^NIsY\xa4ؙ)-M\xca\xcaļ\xa7&\xb2\xf4\x13U&\x80\xa6\xa4\xafL\xa4\xa7L@\x9cMZIMJ\x99\x80\xbdp\xec\xceJ\xc9̏\xf1\x97аx\xbe\x89\xbf\x95D\x9d\xbb0\xa5{\xe6⒋\xf2y\xd0\xdd\U00072d9a\xe6\xcdϘ\xe5\xc9\xed\xfet\U000f3a04奠\xfb\x8c\x03ϣ^\xb3\xdd\xe3\xb1y\xd7\xfa\x9b\xa2w^\x9b#A\xfa\xfc\xa5\x11\xcf\xf5\xc0\x88f\x06\x9eQ\b`1\xe1\x1a\xad<\xf3\x8f\xf93\xb5B\xa7\xf3݆\v/vÛ\xff+/\xc1\xf4\x94-\x16\xf2\xb5{,\x1c\x94\xfa\xe9\xef\t\xfe\u05fc\x81\xe8mY\xfa\xf6\x97\n\xf5\x11\xd4\x01uk1,<\xa4\xf0\x1b\xcd8\xef\xa6\xde\xfaA\u007f\xf8\x12\x12\x03ù\xddp\xf0N\xfa#,\nv\x80#\xc1q{^4\xbcv\xea\xcd\xf9\x01\x13]\xe3\x91\x1fՌ\x8e\xfc\xbed{\xa6\xbeBx]\xd7\xe1t\xe7a\xf1\xd8~\x15\a\xe2|\x17b\x06dꫂ\xb4\x1b\xb8\xc5W\x04\xaf\xe5J,9\x13\xc9VT\xda+\x81\xd7x\x1dp«\x80\x13\x9c\x8a\xd3܊d2\xa5d\xff\xbf\x8as\xf1\x8a\xee\xc5k8\x18\xe7\xb9\x18\v \aY\xfd)\xf9\xfaI\xb7\xcb\xc9\x17,)\xb7\xc3\xcbw \xf3y\xf8\t\xf9\xf7\t\xb7#K\x98&\xe4ٟ\x96_\x9f@\xc3Wr>^\xc9\xfdx\r\a\xe4u]\x90E'dQrf\u007f>;\xbc\xaet\x8ez\xf66\"U\xd4f\x85l\xe0/\xf4\xe7\x1c\xc4\xe6\xeb\x127\xaeW\xcf4\x8d\x85\x94\x9b\xe7\xaf\x19\xfc\xc2e\xee\xf9ᄪs\x8e\xf7\xaeHZ\xc3\"\x1e\xa1o\xad\xb6P\x02\xcc߫\x18,\x99\xa6\xbaq\x9b\xa3\xbf\x9b5k\xf8\xc0\xb2\xfd\x00\xfa>\xea'l\x95.\x98\x85\xcb\xe6R\xea\xda\x03w\xff\xbe\\\x03|Tͭ_\xb7\xa4\x84\xe1E)\x8e\xce\x0f\x88\xc0\xbc\xec\x828O \xa2\xc2dB\xbd\xaaP\xc0g\xc1\xf7{\xe8\xf7\x8e\xdcfֵ\x8bj\xb8&\xee\xf80y\x84\xfb\xafd\x9dPŔ\xac\xad\x1e\x13\xec\x8f\xda\xfb\x1b\x16\x97\xf9\xf9\xe5\xef5\x8dU\x9a\xed\xf0W\xe5K\x90-Ѡ\u07fbW\u007f.h\x8d:Ϡ~\x86\x12;MC1\xb4\x01\xb06}hT\b\xcba\x19S'3;\xd1Z\xb1\xb0\x98\xc7\xc7_\xfd\x02,/p\xfd\xbe\xf2\x17\\\xab\x92i\x83\x8e\x9a\xf5\xc2\xfc\xa0\x8d\xfb߽z\x8eE9TX\xf3\xcfC\xbc5R\x8a\x12]U\x9f\x84\xfd\xa1WP\xad&ђ\x88~\x8d\x8f\xea\xb8h\x1d&\xf9\xdd\x1e\x95\xd0)8\x9d\x9a\x92\x14\xbc\xa0'F/[\xf1hJ\u007fOUݣJs\xcbu\xf7|A\xbaPe3$\xbaU\x9a\xca\x15\x85buT\xde\xe7\xac\xd2{ts\xdaޘ6\xf7\xb1杵\xce\xe0\x8c\x19a}\x0fu\x11@\xad߭\xb2L\x80\xac\x8aM4\xa2\xe4\xd6\xd0\f!\xb4for}\x82\xd1\f\v=ѹ\xb4\xb8\x1b\xcd7\xb9\xeaې\xa6t\xf6\xaa\x1b\x00\xe9\xab6U\x96\xa11\xdbJ\x88c\x93'\x95H\x82hhꥉ\xf2\x91qq>E\xfc\xe8\x189&N\x98SY\x1f*c\xa0\xcc\xc3֎\x1e[>\x8f\xf04\x8a\x04f\xf4\xca\x02\xcfS\xe1v<\x82\x8a\xbf꼓\xd0Д\a|f\xa6ex\xcc\xdel\xc1\xf9\x91\xe4\xf09h\x98\x03\x1eP\x82\x92\x94\aGU\xb2|\x81\xe2\xe1\x98\b\xd4.\x94\x90hW\x95B\xb1\xbc>\xfej\xd3.\x14\xb5}\xa4\xc3]\x1fP\xbf130\x9b\x82\x8f\x11\"\x8c\xad-ok\xdd@\xce,\xae\xa2@\x93\f\x83\xa8&\xce\f\xef\x9f\x02\xc9\xda\xec\xf6\xe1nj\xe4\x84\x04\xb7J+ƿQy\xd1\xefT\\㕥j\xac\xf1\xcafU\xd5P1E\x16ת\xaa\x17_&\xed\xd5\xc5Z\x88\xd4\xc9\xdb\a\x940\\\x97\xdb\xf4\x19\xc3\x05\x1a\xc3vu\x11\xc4gg\x9e\xedP\"y\a\xb1\xb0\xbc\x8f\x9f\xb4\x19\xa6\xfd\x12\x80>\xd0\xcb2[\xb10A\x9d \xd3\xe9\xf5&\xe6T\b\xb5\xf3\x85Ry]&\xba\xb6[O\xa4ɷ\x92\xeb\x14;\xf7C\xd3\xd1ц\xeeh\x88\x11mYo\x14|ǝ\x91蘴cz\xc3v\xb8ʔ\x10H\xbav\x8c\xd7kn\u0590\xc7\xfb\x05\x99Y\\\xda\xc7n\xdf\x10\x10\xf4\xdc\xf6\x15t\x98\xcf\x15\xa3*ϖkl˦\x8f\x10R4\xf1Iv\xad\xa7B\xb4\xc0\xf8\x18\xd3n\xdfz\x83\x05\xbd\x1a\xdc\xccPo\xfc*xJ\xf1\xa0O\xc1~S\xfa\n\n.\xdd\u007f\x9cSL\x11\xbbz\xf0I\xf8Sm\xcb\x05\xbc\xef]\x9f\xe69E\xc7\xca\xc4zCL\xf9q\xf1\x14\xfa\x15|±\xdb\xe1\xb3\xe21\xa7\x18u\xac\xaa\xba\xebr'\xef\xb5\xdai4\xe3]\xb5\x82?3n\xb9\xdc}T\xfa^T;.[{\xe3\xa4\xce\xf7L[΄8z|b\x88r\xc9\x04\xff+\x97\xbbw[\x8b)\xb3-\x8cX\x9e\xb2Q̑\xdf\x12\x10\x9e\xfa\xe1=\xbaCy\xd2m\x89\x8bL\xe0\xc0\x92Ԅnm\xf4\x8dK/唫\xbfQ\x95\xed\xa9\xc9V\xcdFD\xbe\x9es\r\x9f\x94\xc5\xfaR\x87\xf7a\xba\x83\x05\x8d]\xe1v\xab\xb4\xf5\xc1\xbe\xd5\n\xf86\xb8A\xb1\xe8\x0e\xe3\x82.\xa5}\x01w\xe0\xb6\xcd\xcbiw&E84)\x18\xaarW\xb0\xa3\xcf\xdbfY\xe6\xbcl\xbc6\x96\x89\x88\xee\xfe\xae4H\xf27\xdd\xce\xc2\xfcO\x11\x1bsD\xf0\xbbn\xff\xa6xFsr\x138O9zW\xe3ϭ\xe8)\x0e\xf4\xda\x02%Jr\xf0\xd5\xd9C\x95A\x12\x83\xb2D\t\xcc\x04|\x12\x1e\x95γu.4i\x99\xb6\xa9\x1e\xd3C\xaf\xf3\x82\xb3D\x90\xe3\xf8>\x84ȯ\u007f\\{;\xfcs.W`\xb8\xac\xff~\x89\x8f+{Q0·\xd2HA\xba\xe8\x1d\xf6\xc8\xfb\xe9\xf9:}\xf4\xff\xb6nΡ9=?\xa4\xd8\xcb_\a\xdd\a/\x14\xa8L\u007f\xd3%ظ\x11z\xfc\x81o\xfd\xb5z\xe6\xb0\xfe\xe3\xdf\xfd\xe5\xc1!\xc9\x1e{3k\x8a\x91\x95\xd5\xd8T\vE\xf9\xef\x05:\x1b\xc9 \xf6\xad\xbc7'\x99\xf3\x87\xf3\x1cԗ\xf4N\xeb?\xad\xf32>\xdb\xe1<\xbf\xf4՜җ]\xdd3\xa3?籴\xc7\xfe\x1c\xbaE\xbc\xd2\x00!\xe2\x97F\x96\xd1x\xaa\x8b~i\xc7-\xadq\x9c\xa8\xf8?pU_\xc81\x8d\x9e\x03\xa3\x8f\xa4@\xf3\xce\xde\x0e3\x85/m$\x9ce\x19:q\xfd4\xfc\x13Z\x97\xfe\xefQ\xd4\u007f%\x8b\xfe\x99)\xe9\x8f[s\x03\xff\xf5?\x17\x10\xaeZ\xbe\xd6\u007f\x0e\xcb}\xfc\xff\x00\x00\x00\xff\xff\xed\xa36\x94nl\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c9n\xef\xfa\x15(\xe5\xc1wU\x9aѺ\xf2\x90\x94\u07bc\xb2]Q\xed\xc6VY:\xdfC\x92\aN7f\x86+6\xd9G\xb2G\x9eK忧\b\xb2\xbf\xd9ݜ\xb1t\xb7we\xbe캇\x04A\x00\x04\x01\x10\x84.V\xab\xd5\x05+\xf9WԆ+y\x03\xac\xe4\xf8͢t\xff2\xeb\xa7\u007f7k\xae\xae\x0fo/\x9e\xb8\xcco\xe0\xb62V\x15_ШJg\xf8\x1e\xb7\\r˕\xbc(в\x9cYvs\x01\xc0\xa4T\x96\xb9\xcf\xc6\xfd\x13 S\xd2j%\x04\xea\xd5\x0e\xe5\xfa\xa9\xda\xe0\xa6\xe2\"GM\xc0\xeb\xa9\x0f?\xad\xffm\xfd\xd3\x05@\xa6\x91\x86?\xf2\x02\x8deEy\x03\xb2\x12\xe2\x02@\xb2\x02o`ò\xa7\xaa4\xeb\x03\n\xd4j\xcdՅ)1ss\xed\xb4\xaa\xca\x1bh\u007f\xf0C\x02\x1e~\r?\xd3h\xfa \xb8\xb1\xbft>\xfeʍ\xa5\x1fJQi&\x9a\x99\xe8\x9b\xe1rW\t\xa6\xeb\xaf\x17\x00&S%\xde\xc0'7E\xc92\xcc/\x00\xc2rh\xcaU@\xf8\xf0\xd6C\xc8\xf6X0\x8f\v\x80*Q\xbe\xbb\xbf\xfb\xfa\xaf\x0f\xbd\xcf\x009\x9aL\xf3\xd2\x12Qѽ#\x90-\v\x1c\xc1\xb8ܢ\xf6L\xdcjU\x10L\x94\xb9\x97>\x12]\xc1Q\x0e\xc9o\xaaM\xc1\xad\xe3\xfb_*4N\xc8\xd5\x1anI\xc5\xc0\x06\xa1*s'\x99k\xb8\x93p\xcb\n\x14\xb7\xcc\xe0\xab3\xc0Qڬ\x1ca\xd3X\xd0Վ\xc3Ξj\x9d\x1fj]6\xc1/\xaf\x10\x1eJ\xccz\x1bƍ\xe2[\x9eѶ\x80\xadҭ\xbe\xf0\xeaj\xdd\x03\x19߲\xaee\x86?HV\x9a\xbd\xb2N\xff\xaa\xca\x0e{\f\x10\xba}\xb8\x1b\f\xa8\x91\t\xa8\x91Z\xa9\f\xe6n\x9f=3n\x1dz#\x98\xe0\x00\xc1W\xd205<\xd24\x95\x01[iI\xbb\xf4\v\xb2\xfc\xf8\xa8\xfed\x10\U0008a135>+\xae`\x83[\xa51\x02W\xa3\x1b\xef:\xa3֎0\x86PR\x95]\xc3\xe3\x1e\x1d\x19Y%l\x90{n\xe0\xedOPpYY\\\x8f\xa0M0\xd8\x13\x85\xc0\xf8\x15\x98G\xf5\xd1xV-\x90\xef\xfdİ\x0e\x11\x9f\xf7h\xf7\xa8\xa1T\xb5\n\x8e\xacr\xcb\x05\x829\x1a\x8bE\xe0x\xad\xf86\x81\xfa$\x14B\x04\x10\x066\xc7\x1a\xe7\xf1:\xddy\xcb6\x02o\xc0\xeaj<\x9d'\xc3F)\x81l\xa8\x84\x87t\xf8\x82\xc6\xf2l\x81\n\x97C2\xf8Q\x11\"\xe8\xf0\x03\xad-B\x87M+k\x96=!\xb0\x9a\x1a\xeep\x10\xa2C\xc4\x1e\x05\xe0\xbf%\xbcw\x9a+s\xfad\x8c-\x04\xcd\xc5Q\x90\xb6\x94\n\x84\x92;\xd4~6w*)K_^\x85\x9c\x1e\xf13\x88\xe9\a\xd2\xf6\x92^m;:tc\xc8\t\xc2\xed\u06dd\xf7\xf0\x1a\xf6p\x03w\xd2\xf9-\x81\x1et#\u09db?\x1f\xfa\xad\xa8\f\x05\x89\xa5\x92+:*ױ\x99<\xb1\x13A*\xdd\xe3\xc8\x18\xb5fR?a\"\xd8Gw\x92\xf8\xf1\xfe\x8eC\xb0\f\xf3:\xc6I\x91yfq\xc73(P\xef\xe6\x0e\x8en+\x9d~OC!Q\xeb\xfav\xa2\x84\xa5\x1d\xedu\v\xaa;\x1a\x82뷕۹\t\xbdjf/v\x9d\b\xc8Ow]^\x11\x1d\xb1d\u007f,R\x97\xe59ݥ2q\u007f\x82\xc6?\x81\x17\xe3\xb3\xdf#\xe6OȂQ\x88\xf4\u007f\xdd1G\x02\xfd\u007fP2\xae\x13\xf6\xf0;\xba\x1a\x15\xd8\x1b\x1b\xa2X\xddi\xdc\f܀\xe3\uf049\xf1UOdq\xca\xe9\x16\x14\xfe Wۑ\xc5r\x05\xcf{e\xfc\x99J\xa1\xd9E\x90\xdc\xc0\xe5\x13\x1e/\xafFz\xe0\xf2N^\xfa\x03\xfedu\xd3X\vJ\x8a#\\\xd2\xd8\xcb\xef1\x82\x12%1\xa9\x1b]A\xa7\x9a\xcaΗ\xac-\x017\xb0\xb9wuf\xee\x1c\xd6IrX*\x13\xb9M\x9a@\xe5^\x19\xeb#\x8b=\xb3\xf4\x94(\x16x\x19\n\xd1+`[\u007f\xf3\xadt}\xa7\xe9\xd4\xde \xe0\xea\xb8f\xe65\xaccc\x13\x11\xf3@\x9dcu\xd9\xee`\xafO/\xfdE'M\xc222.\x16\xe1\x96Zeh̼\x88$h\xeb\x85 a\x13 dށ\xf1\x17\x86\xf3Aɺ\xa5\x1b\xa4\x8eH'\x9a\xf2\x1f\xbeu\xa2\x97n\xf3\xbb\u007f/\tߩx\x01\xed٢`Û\xf1$\x14o\xfd\xc8z\x9b\x04@\xde5л\x8a\xb6z\xba\x05\x19\x04\xe9\xf7pL\x17\\\xde\xd1\x04\xf0\xf6ŏ\xf5FI\xe29\x86\xfbm=\xb6%z\xf3\x81vo\xaaE\xa4(r\xaf\xb1ǹq\x9c\xdb\x19\x8a\x89 \xa5\xb2\xddp\x82\x83[\xaa\xfc\x8d\x81-\xd7\xc6v\x11M\x15\x8aja\xf7\xb7\xedT\xcfI~\xd0\xfa,\xc7\xe9\xb3\x1f\xd9\td\xed\xd5s\x9d_0y\x15\x1bkt)\x84\xc0\xb7\xc0-\xa0\xccT%)\xfc\xe2\xb6:M\xe1Y\xe0\x15t2\xc9\xd2\x14\x84k(\xab\"\x8d\x00+\x92:.g\xe34\xdd\xee\x1f\x19\x17\xaf\xc16;\x95\x86\x11k=\xb6\xd5\xf9\x18\xddD\x91\x82}\xe3EU\x00+\x1c\xe9Sݞ\xad\xcf\xe2\xe8q\xbc\xc9\xe5 \xb8t\x8cX\xe56U)Ц\xeeH\x9f\xb5ᶉ\xe196\as\x90\x02%\x81\xc1\x96q1qy>n'\xd1\xf6\x14_#(\x8b\x97s\"\xd2&_\x11)\x12\x02\xb1\x89\xc6⼶.u\xba\xa9x\xaf1\xcd<[\nJ\xd7\xe6Y\xa9\xb9\x93%\xf5\xd2\x16Z\x101&\x8f?L\xb4Q\xfba\xa2-\xb4\x1f&\xdad\xfba\xa2-\xb7\x1f&Zh?L\xb4\xba\xfd0\xd1~\x98hs\xdd\xe6\xb4\xf5\x12F\xfe\xc5\xc9ď\x8bX$\\Oϡ8\x03?dS\xdc\xfa\xd7'\xa9\x19\x96w\xf1Q\x91\xac\xe0\xf0\xaceE/rb\x12\xd0&]\xb4GI\x93r\xe96H-\xde>\x81~!\t\xf3;\xb2oS\x12~\x96\xd2|\xfay\xa6M\x9aM\x9dh\xaa\xeaI\"t\xa8_\xf68\xb3\xb7\x9bC\xd2\xcf\xd7!;\xb7\xc6\xf4\uf783\x9a\x90\x8a\xb3\x90\x803\x9f\x98;G\xaf\x81\xeb\xd1'\x98\xee%\x8c\xfe~\xe8e\xb1\xf8\\\x06QN{\xc4q\x17\x19\xb2\xf4\x8c#\xb2\x1e\xf2Q\xccQf{\xad\xa4\xaaLpk\x1c\xf4w\xe4]\x85\x1b\x17\xca6\x89\xbd\xbd\x88\xc0|\v{UE\x12Cgh\xb7\x90&4\x9d\x1c\x14\xae\xc2в\xc3\xdbu\xff\x17\xabB\xaa\x10\xecɊi\xde\xffJK&:;\x85\xa8\x9f\"4qF\x9dzg\x96\x9e)\x9d\x9e$4\x9f\xd5sJj\xd00\xf1g\x12\xe8rBP\x8a뼐\xfcsF\xcaOb\xba\xe7w\xdf\f\xa6$\xf5\x9c\x95ʳ\x98\x11\x99\x98\xc0\xd3O͙\ayB\xdaN\x12q\x96StNN\xcc\t\x890\xb3\xebHNlj$\xda\xcc\x02\x9eL\u0099K\xafY\bˍSoғjfAS\xc2\xcdr*\xcd\xcb%̾\x84\x130\xadj\x16\xd3a\x16\x9d\x84y\xfc\x16\x13^NIsY\xa4ؙ)-M\xca\xcaļ\xa7&\xb2\xf4\x13U&\x80\xa6\xa4\xafL\xa4\xa7L@\x9cMZIMJ\x99\x80\xbdp\xec\xceJ\xc9̏\xf1\x97аx\xbe\x89\xbf\x95D\x9d\xbb0\xa5{\xe6⒋\xf2y\xd0\xdd\U00072d9a\xe6\xcdϘ\xe5\xc9\xed\xfet\U000f3a04奠\xfb\x8c\x03ϣ^\xb3\xdd\xe3\xb1y\xd7\xfa\x9b\xa2w^\x9b#A\xfa\xfc\xa5\x11\xcf\xf5\xc0\x88f\x06\x9eQ\b`1\xe1\x1a\xad<\xf3\x8f\xf93\xb5B\xa7\xf3݆\v/vÛ\xff+/\xc1\xf4\x94-\x16\xf2\xb5{,\x1c\x94\xfa\xe9\xef\t\xfe\u05fc\x81\xe8mY\xfa\xf6\x97\n\xf5\x11\xd4\x01uk1,<\xa4\xf0\x1b\xcd8\xef\xa6\xde\xfaA\u007f\xf8\x12\x12\x03ù\xddp\xf0N\xfa#,\nv\x80#\xc1q{^4\xbcv\xea\xcd\xf9\x01\x13]\xe3\x91\x1fՌ\x8e\xfc\xbed{\xa6\xbeBx]\xd7\xe1t\xe7a\xf1\xd8~\x15\a\xe2|\x17b\x06dꫂ\xb4\x1b\xb8\xc5W\x04\xaf\xe5J,9\x13\xc9VT\xda+\x81\xd7x\x1dp«\x80\x13\x9c\x8a\xd3܊d2\xa5d\xff\xbf\x8as\xf1\x8a\xee\xc5k8\x18\xe7\xb9\x18\v \aY\xfd)\xf9\xfaI\xb7\xcb\xc9\x17,)\xb7\xc3\xcbw \xf3y\xf8\t\xf9\xf7\t\xb7#K\x98&\xe4ٟ\x96_\x9f@\xc3Wr>^\xc9\xfdx\r\a\xe4u]\x90E'dQrf\u007f>;\xbc\xaet\x8ez\xf66\"U\xd4f\x85l\xe0/\xf4\xe7\x1c\xc4\xe6\xeb\x127\xaeW\xcf4\x8d\x85\x94\x9b\xe7\xaf\x19\xfc\xc2e\xee\xf9ᄪs\x8e\xf7\xaeHZ\xc3\"\x1e\xa1o\xad\xb6P\x02\xcc߫\x18,\x99\xa6\xbaq\x9b\xa3\xbf\x9b5k\xf8\xc0\xb2\xfd\x00\xfa>\xea'l\x95.\x98\x85\xcb\xe6R\xea\xda\x03w\xff\xbe\\\x03|Tͭ_\xb7\xa4\x84\xe1E)\x8e\xce\x0f\x88\xc0\xbc\xec\x828O \xa2\xc2dB\xbd\xaaP\xc0g\xc1\xf7{\xe8\xf7\x8e\xdcfֵ\x8bj\xb8&\xee\xf80y\x84\xfb\xafd\x9dPŔ\xac\xad\x1e\x13\xec\x8f\xda\xfb\x1b\x16\x97\xf9\xf9\xe5\xef5\x8dU\x9a\xed\xf0W\xe5K\x90-Ѡ\u07fbW\u007f.h\x8d:Ϡ~\x86\x12;MC1\xb4\x01\xb06}hT\b\xcba\x19S'3;\xd1Z\xb1\xb0\x98\xc7\xc7_\xfd\x02,/p\xfd\xbe\xf2\x17\\\xab\x92i\x83\x8e\x9a\xf5\xc2\xfc\xa0\x8d\xfb߽z\x8eE9TX\xf3\xcfC\xbc5R\x8a\x12]U\x9f\x84\xfd\xa1WP\xad&ђ\x88~\x8d\x8f\xea\xb8h\x1d&\xf9\xdd\x1e\x95\xd0)8\x9d\x9a\x92\x14\xbc\xa0'F/[\xf1hJ\u007fOUݣJs\xcbu\xf7|A\xbaPe3$\xbaU\x9a\xca\x15\x85buT\xde\xe7\xac\xd2{\x9b沴\xb9\x8a5\xef\xacu\xb6f\xcc\xfe\x8a\xa071\xb6\xd6\xeaVY&@Vņ\x8e\x89\x98Ji\x86\xd05\xee\xec\xfd\xadO+\x9aa\x9c'5\x97\x16w\xa3\xb8Ul\xad\xb7!/霵6c\xd3\xd7j\xaa,Cc\xb6\x95\x10\xc7&'ꔅG`\xbe\x14)>2.\u03a2\x83\x1f8A\x04\xbf\xb6I=\x9a\xc4\xe6P\xfb\x02e^o\xde\xd1Q\xe0\x1ae\n\x9eF\x87\xc0\x82^\xe1\xdfy\x02\u070eGPyW\x9dwR\x16\x9a\x02\x80\xcf̴l\x8eY\x94-8?\x92\\:\a\rs\xc0\x03JP\x922ݨ\x0e\x96/A<\x1c\x13\x81څ\x12R\xe9\xaaR(\x96\xd7\a\\m\xbc\x85\xb2\xb5\x8ft|\xeb\x03\xea7f\x06fS\xd21B\x84\xb1dzk\xea\x06rfq\x15\x05\x9at\xf4Gumfx_\xcf'+\xadۇ\xbb\xa9\x91\x93\x12\\w\x88\xf1oT@\xf4;\x95\xd4xe\xa9*j\xbc\xb2%\x05\xd5SG\x91ŵ\n\xeaŗI{u\xb1\xda!u\xf2\x16\x00\xa5\x04\xd7\x055}Np\x81ư]]\xe6\xf0\xd9\x19`;\x94H\xf6\u007f,\xf0\xee#$m\x0ei\xbfȟ\x0f\xe5\xb2\xccV,LP\xa7\xc0tz\xbd\x89)`\xa1v\xbe\x14*\xaf\vAז\xe9\x894\xf9Vr\x9db\xc9~h::\xda\xd0-\f1\xa2-܍\x82\xef\xb83\x03\x1d\x93vLo\xd8\x0eW\x99\x12\x02I\u05ce\xf1z\xcd\xcd\x1a2u\xbf 3\x8bK\xfb\xd8\xed\x1bB~\x9e۾F\x0e\xf3\xd9`T\xc7\xd9r\x8dma\xf4\x11B\x8a&>\xc9r\xf5T\x88\x96\x10\x1fc\xda\xed[o\xb0\xa0W\x83#\x19*\x8a_\x05_(\x1e\xd6)\xd8oJ_A\xc1\xa5\xfb\x8fs{)&W\x0f>\t\u007f\xaa^\xb9\x80\xf7\xbd\xeb\xd3<\x98\xe8ؑXo\x88)O-\x9e$\xbf\x82O8v,|\xde;\xe6\x14\x85\x8e\xd5Mw]\xee\xe4\xbdV;\x8df\xbc\xabV\xf0g\xc6-\x97\xbb\x8fJߋj\xc7eko\x9c\xd4\xf9\x9ei˙\x10G\x8fO\fQ.\x99\xe0\u007f\x8dq\xa7\xfb\xe32\xa0F\xddF~K@c\xea\x87\xf7\xe8\x8e\xdaIw#.\b\x81\xaeK\xb2\x10\xba\xb5Q3.\xbd\xecR\x8e\xfdFU\xb6\xa7\xfcZ\xe5\x19\x11\xe4z\xce5|R\x16\xeb\xcb\x18އ\xe9\x8e\v4v\x85ۭ\xd2\xd6\a\xe9V+\xe0\xdb\xe0\xbeĢ2\x8c\v\xbaL\xf6\x85ׁ\xdb6\x9f\xa6\xddo\x14\x99Ф6\xa8:]\xc1\x8e>ߚe\x99\xf3\x8e\xf1\xdaX&\"\x1a\xf9\xbb\xd2\x17\xc9Ot\xfb\x05\xf3?E,\xc7\x11\xc1\xef\xba\xfd\x9b\xa2\x17\xcdyL\xe0<\xe5\xe8=\x8c?\x8d\xa2g3\xd0+\t\x94𬹵\xee\x04\xe8\u07b6\x83u:_\b0N\vN\x948\x9d;\x8b\xe8wg-\xdcMG\xff\xfb\x01\x89\xa6\xf3\x94\xb1\x11\x16\xa7\x1c[6D\x82\x89e\xf9\xf7\x96\xdc\xd4c\x1d+\xb3=\x93;'TZU\xbb}-\x97\x13g\xf9T\xf0\xbcrHAI\x1a\xc2\xd47\x9d\xb6Ҳ\x13=\rw\x9fy\a]\x96=Mb\x1ans\xea?\xfeq\x1d\x8a\x96\xae\xb6Z\x15\xab\xc0\v\xba\xa0\xbc\n\x11M͕s\xd8\xed>Jr\xf0U\xd5Cu@\x12\x83\xb2D\t\xcc\x04|\x12\x1e\x83γu.\xa4h\x99\xb6\xa9~\xd0C\xaf\xf3\x82\vD\x90\xe3\xf8>\x84\x88\xad\u007f\x14{;\xfc3,W`\xb8\xac\xff\ue20f\a{Q0\xce3\xd2H\xc1\xb5\xe8\xdd\xf3ȧ\xe9y0}\xf4\xff\xb6\xceˡ9\x13?\xa4X\xc1_\a\xdd\a/\v\xa8\xbc~\xd3%X\xae\x11z\xfc\x81o\xfdux\xe6\xb0\xfe\xe3\xdf\xfd\xc5\xc0!\xc9\xcaz3k`\x91\xed\xd4XJ\v\xc5\xf4\xef\x05:\xcb\xc7 \xf6m\xb77'\x19\xe9\x87\xf3\xdcΗ\xf49\xeb?\x89\xf32\x9e\xd8\xe1\x8aU_\xbdٓ\xe9t:a\x8d|F\xeb\xa4\xd1s`\x8dğ=j\xfa媗\u07fbJ\x9a\xd9\xfe\xfbɋ\xd4b\x0e\xcb༩\xbf\xa23\xc1r\xfc\x84\x1b\xa9\xa5\x97FOj\xf4L0\xcf\xe6\x13\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B;ݢ\xae^\xc2\x1a\xd7A*\x816\x12/W\xef?T\xbf\xab>L\x00\xb8\xc5x\xfcI\xd6\xe8<\xab\x9b9\xe8\xa0\xd4\x04@\xb3\x1a\xe7\xb0f\xfc%4\xce\x1b˶\xa8\fOwU{ThM%\xcd\xc45\xc8\xe9\xea\xad5\xa1\x99C\xbb\x90(d\xb6\x92H\x1f#\xb1U\"v\x9f\x89\xc5u%\x9d\xff\xf3\xe5=\xf7\xd2\xf9\xb8\xafQ\xc12u\x89\xad\xb8\xc5\xed\x8c\xf5?\xb4WOa\xedTZ\x91z\x1b\x14\xb3\x17\x8eO\x00\x1c7\r\xce!\x9en\x18G1\x01ȘEjS`BD-0\xf5h\xa5\xf6h\x97F\x85Z\x9f\xee\x12踕\x8d\x8f('Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6`\xb1gR\xb1\xb5\xc2\xd9_4+\xffGz\x00?9\xa3\x1f\x99\xdf͡J\xa7\xaaf\xc7\\YM:z\xec\xcc\xf8#\t༕z;\xc6\xd2=s\xfe\x99))NZ\a\xe9\xc0\xef\x10\x14s\x1e\x8c\xb6e,\x1e?\x97ț\x1c(\xfb[ƪ\x82E\xf6\\\xb3\x81\x0f \xa4\xa3\x02\xc0E\xa2C\xb0\xa8<\xa3\xf59x\x1b\xde$>7z#\xb7C\xa1\xbb5\xcd%\x8b\xb9B\xba\x87\xdc2\xdeD\xa1\x89\xac\xa3\xb1f/\x05\xda)\xf9\x87\xdcH\x9e9\t6e\xae\x8dD%\xdcP\xd2\v^\x16E\xb1(ȫ\x99\xba\xa2\xc3\xe5ic,\x8d\x99\xd4ɂ[\x021\xd8\xd8:\xa7T\xedQ\x8bS5rƍ\x89Qˡ\x80\x83\xf4\xbb\x14\x0e\u0558\xdf\xc1\xab\xbeG\xe3\x05\x8fc\xd3=ޟvH;S\x02Ep\xc8-\xfahm\xa8\xc8|Ȕ*\x80/\xc1ŀڏ\x13e\xc4B\xad\x9c~\xc1\xe3\x10h\xb8\xa6\xdc\\\xc2\\g\xf9\x8eJ\xe7°\xc5\rZ\xd4~4\xa8Sgb5z\x8cq]\x18\xee(\xa4sl\xbc\x9b\x99=ڽ\xc4\xc3\xec`\xec\x8b\xd4\xdb)\x01>\xcd\x1e4\x8bm\xc5\xec\xbb\xf8\xe7\x82\xc8O\x0f\x9f\x1e\xe6\xb0\x10\x02\x8cߡ%\xadm\x82*\x86֩o\xde\xc7\x1c\xfb\x1e\x82\x14\u007f\xb8\xfb\x16\\L\x93<\xe7\x06lV\xd1\xfa\x8fT\xa8E\xa6\b\xa2UҊ\xb1@\x99\x92\x94]gm\xa6X3f\x88c\x15fwP`\xa2\f2\x16Q_p\x18L_q\xb3\\\xec^\xf1\xb1RHK-$\xa7B\xec\xdc7J\x83!\xce\xea\xed\x11\xc1\xfa\x15\xf8\xa5\x880.x\x12 \xe7\xc3+\x1c?t\xf7\xb6mY\nO9\xc79\xf4T@9\xd0H9\x90\xd9!r1(p\xa35y\xa37\xc0N\xa1\xee\xce\xf5c\xfc\x1b#\xc4:\xf0\x17\x1c\x01~ \xcaǸ\xb1`\x9c\x8e\x11/\xc1a\f\xbe\xd7\u0600\xeb6\xce\xd9\x12\xed-\xbc,\x17\xb4\xf1\x94&\x19,\x17\xb0\x0eZ(,\x1c\x1dv\xa8\xa9C\x90\x9b\xe3\xf8]4\x9e\xeeW\x05\xd5Xa\xe4\x1a\xbf`;.C\x8a\xe1sX\x1fGj\x82\x1b\x84l,n\xe4\xcf7\b\xf9\x187\x16\xc0\x1b\xe6w \xb5\x93\x02\x81\x8d\xc0\x9f\x8a\xb5\v\x82\x9e\xf2\xffC\x8e\"ߠ\x9e\u05fc=\xb1\xf3\x16\x87/\x18_\xf1\x9fǼ\xed\x84B\xf9\x9d#\xffy-xɏG%ڟ\x1e\f\xfe\x94*,>\x92*Ϙy\x1e\x9ex\xa5R+\xcf\x16c\xceLu\x81\xb1\x16]c\xb4\xa0\xe6\xe9\xb6:\xade\xf9\u007fW\xad\x8d\xabuz\x1e\xe5zkE\v7\xb5*\xf1\x89\xe6\xcd\xcdJz\xb8\xea\xb6\x02f\xed\xa8Sl\xfb\x95\x9e\x8c\xbfH\x9b\xf2\xaeӧP?\xac!\xe8X\xa9Ō_\xc1\xdf5|\xa2ޖ\xb2\x93\x98\x13\xdfv\xcc\x00\xa4\x03m\x0et\xbcC/\x92\x00\xa3S\xbe\xa6n\x8di\x91\x9b\xe1\xb8t\x90JQƶX\x9b\xfdhƦBӢ:\x02sd:\xfb\xdfT\x1f\xaaw\xbfZ\x17\xa4\x98\xf3\xd4Ԡ\xf8\x8a{9|\xe5\x19\xa2{?8Q\x1c\xff\xe4\x0e\xf4\xe3\xc7\xd2,\xcfl\xde\xf6\xe3\b\x18\x1b\xa9\xa8\x16\x1c\x89\x13m\xc50|\x8f\xfc\xb8\xba\xbfs\xb1\x84G\xed\xc7ʾ\x03Z\x8c\x1d\x13\n\xaa\xe2M~\x97\bΣ\x1d1\x80\x93\xf6\xa2\xceA\x19\xbd\xed9N\x1a\xf9\x95\x82*\xb4dPƂ@O\xa9Io\x81\xef\x98\xdeb\xfb\n\x95\xf9\u007f\x9dS2\x9f\x9eʹ\x16\"\xf5%\xf3\xb8I\xa3Or\xacL\x1f\xbc\x00\xb7\x9b\xc7_\u007f\v\xf7E\xb3\x17ۜ+\xb8\x0f\xf6\x97,M\xa0N}\xfb\"\u070eooo\x87\xcf\xcd7 \xf1ַ\xf0W\xde5\xe0\xc0\\\xfb*\xfe\xeb\xe1PS\xb5z\xb5\x04\xfe\x92v\xa5\xe7\xc3|\x04\xd8\xda\x04\xff\x9agލ\x19t~\xee\u007f\v\x8f\xf1#Ƶ\"\x83\xf6\x14\x8d\xf0`\xa9\x95l_\xc5bP\x18\xcb-\xb7?/-z\xdfZ\xbak\xc3/17\xc85\x9ak\a\x93)_v\xf4\x9aA\xee΄\xf5\xe9\xa5x\x0e\xff\xfeϤMה\x13\x1b\x8f\xe2\x87\xfeǵw)d\x94/d\xf1'\xa7:&}\x1d\x84\xbf\xfdc\x92\xaeB\xf1\\>i\xd1\xe4\u007f\x03\x00\x00\xff\xff\x1d\r\x93\v\x97\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96Ms\xe36\x0f\xc7\xef\xfa\x14\x98}\x0e{y$\xefN\x0f\xed\xe8\xd6\xcd\xee!\xd36\xe3I2\xb9tz\xa0I\xd8\xe2F\"Y\x00t\xeav\xfa\xdd;$%\xbf\xc8v6=\x947\x91 \xf0\xe7\x0f\x04Ī\xae\xebJ\x05\xfb\x84\xc4ֻ\x16T\xb0\xf8\x87\xa0K_\xdc<\xff\xc0\x8d\xf5\x8b\xed\xc7\xea\xd9:\xd3\xc2Md\xf1\xc3=\xb2\x8f\xa4\xf13\xae\xad\xb3b\xbd\xab\x06\x14e\x94\xa8\xb6\x02P\xceyQi\x9a\xd3'\x80\xf6N\xc8\xf7=R\xbdA\xd7<\xc7\x15\xae\xa2\xed\rRv>\x85\xde~h\xbeo>T\x00\x9a0o\u007f\xb4\x03\xb2\xa8!\xb4\xe0b\xdfW\x00N\r\u0602\xc1\x1e\x05WJ?\xc7@\xf8{D\x16n\xb6\xd8#\xf9\xc6\xfa\x8a\x03\xea\x14xC>\x86\x16\x0e\ve\xff(\xaa\x1c\xe8sv\xf5)\xbb\xba/\xae\xf2joY~\xbaf\xf1\xb3\x1d\xadB\x1fI\xf5\x97\x05e\x03\xb6n\x13{E\x17M*\x00\xd6>`\vwIVP\x1aM\x050\xf2\xc82kP\xc6dª_\x92u\x82t\xe3\xfb8Ldk0Țl\x90L\xf0\xb1\xc3|D\xf0k\x90\x0e\xa1\x84\x03\xf1\xb0\xc2Q\x81\xc9\xfb\x00\xbe\xb2wK%]\vM\xe2\xd5\x14\xd3$d4(\xa8?ͧe\x97\x04\xb3\x90u\x9bk\x12X\x94D\x9eD\xe4\xb8\xd6;\xa0#\xbe\xa7\x02\xb2}\x13:ŧ\xd1\x1f\xf2µ\xc8\xc5f\xfb\xb1\x90\xd6\x1d\x0e\xaa\x1dm}@\xf7\xe3\xf2\xf6黇\x93i8\xd5z!\xb5`\x19Ԥ4\x81+\xd4\xc0;\x04O0x\x9a\xa8r\xb3w\x1a\xc8\a$\xb1\xd3\xd5*㨪\x8efg\x12\xde'\x95\xc5\nL*'\xe4\fm\xbc\x04hƃ\x15\x98\x96\x810\x102\xbaR`'\x8e!\x19)\a~\xf5\x15\xb54\xf0\x80\x94\xdc\x00w>\xf6&U\xe1\x16I\x80P\xfb\x8d\xb3\u007f\xee}s:g\n\xda+9\xe4g\x1a\xf9\xd29\xd5\xc3V\xf5\x11\xff\x0f\xca\x19\x18\xd4\x0e\bS\x14\x88\xee\xc8_6\xe1\x06~I\x98\xac[\xfb\x16:\x91\xc0\xedb\xb1\xb12u\x13\xed\x87!:+\xbbEn\fv\x15\xc5\x13/\fn\xb1_\xb0\xddԊtg\x05\xb5D\u0085\n\xb6\xce\xd2]\xee(\xcd`\xfeGc\xff\xe1\xf7'Z\xcf.H\x19\xb9\xd0_\xc9@*\xf3\x92\xf6\xb2\xb5\x9c\xe2\x00:M%:\xf7_\x1e\x1ea\n\x9d\x931\xa7\x9f\xb9\x1f6\xf2!\x05\t\x98uk\xa4\x92\xc45\xf9!\xfbDg\x82\xb7N\xf2\x87\xee-\xba9~\x8e\xab\xc1\nOW2媁\x9b\xdcbSQ\xc7`\x94\xa0i\xe0\xd6\xc1\x8d\x1a\xb0\xbfQ\x8c\xffy\x02\x12i\xae\x13ط\xa5\xe0\xf8\xef07.Ԏ\x16\xa6\xf6}%_\x17\x8a\xf6!\xa0N\x19L\x10\xd3n\xbb\xb6:\x97\a\xac=\xc1Kgu7\x15\xed\x8c\xee\xbe\xc0\x9b\x93\x85\xcb\x05\x9dơM\xceW\xae\x1e\x1er\xee,\xe1\xec\x16\xd6p\xd6s_璛\xe1\xbf$S:\xf1\xc8FG\"trԟեMoe\x81D\x9e\xcefg\xa2\xbed\xa3\xfc\x04P\xd61(\xb7\x1b7\x82tJ\xe0\x05)\x95\x81\xf61\xf5\x194`\xe2\x19\xbf\x11\xcb\xf1\xbf$\x90\xd7\xc8ܜ\xd9Y\xc1ႦW\xb2\x93Fz^\xa8U\x8f-\bE\xbc\x92YE\xa4v\xb3\xb5\xfc\xcf\xfa\x06\x82e\xb2\xb9\x94\x83\xfd\u007f\xfa\x9bIȸ]\x1c\xce#\xd5p\x87/\x17foݒ\xfc\x86\x90\xe7W>-.\v\xbd\xfdc\xe0\r\x94.^ʳIN\xfd\xce\x1cQd\xf1\xa46\xc7\\9\xae\xf6\xfd\xbb\x85\xbf\xfe\xae\x0e\xf7Zi\x8dA\xd0\xdc\xcd_i\xefޝ<\xb7\xf2\xa7\xf6\xae\xbc\x8c\xb8\x85_\u007f\xabJ(4O\xd3\xeb)M\xfe\x13\x00\x00\xff\xff--\nM\xde\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WO\x93ܦ\x13\xbdϧ\xe8\xf2\xef\xe0_\xaa,\x8d]9$5\xb7d\xed\xc3V\x1cǵ\xe3\xec%\x95\x03\x83z$\xb2\b\b\xdd\xccz\xf3\xe9S\r\xd2\xfc\xd1hfׇp\x134\xcd\xe3\xf1\xfa\x81\x16UU-T0\xf7\x18\xc9x\xb7\x02\x15\f~et\xf2E\xf5ÏT\x1b\xbfܽ[<\x18\u05ec\xe0&\x11\xfb\xfe\x0eɧ\xa8\xf1=n\x8d3l\xbc[\xf4ȪQ\xacV\v\x00\xe5\x9cg%\xdd$\x9f\x00\xda;\x8e\xdeZ\x8cU\x8b\xae~H\x1b\xdc$c\x1b\x8c9\xf9\xb8\xf4\xeem\xfdC\xfdv\x01\xa0#\xe6\xe9_L\x8fĪ\x0f+p\xc9\xda\x05\x80S=\xae\xa0\xf1\x8f\xcez\xd5D\xfc;!1\xd5;\xb4\x18}m\xfc\x82\x02jY\xb4\x8d>\x85\x15\x1c\x06\xca\xdc\x01P\xd9\xcc\xfb!\xcd]I\x93G\xac!\xfeen\xf4\xa3\x19\"\x82MQ\xd9s\x10y\x90\x8ck\x93U\xf1lx\x01@\xda\a\\\xc1'\x81\x11\x94\xc6f\x010\xec=ê\x86\xdd\xedޕT\xba\xc3^\x15\xbc\x00>\xa0\xfb\xe9\xf3\xed\xfd\xf7\xeb\x93n\x80\x06IG\x13838\xc1\f\x86@\xc1\x80\x00\xd8\xefA\x81r\xa0\"\x9b\xad\xd2\f\xdb\xe8{\xd8(\xfd\x90\xc2>+\x80\xdf\xfc\x85\x9a\x81\xd8G\xd5\xe2\x1b\xa0\xa4;P\x92\xaf\x84\x82\xf5-l\x8d\xc5z?)D\x1f0\xb2\x19Y.\xedH\\G\xbd\x13\xe0\xafeo%\n\x1aQ\x15\x12p\x87#?\xd8\ft\x80\xdf\x02w\x86 b\x88H\xe8\x8a\xceN\x12\x83\x04)7젆5FI\x03\xd4\xf9d\x1b\x11\xe3\x0e#CD\xed[g\xfe\xd9\xe7&aH\x16\xb5\x8aG9\x1c\x9aq\x8c\xd1)\v;e\x13\xbe\x01\xe5\x1a\xe8\xd5\x13D\xcc<%w\x94/\x87P\r\xbf\xfa\x88`\xdc֯\xa0c\x0e\xb4Z.[\xc3cQi\xdf\xf7\xc9\x19~Z\xe6\xfa0\x9b\xc4>Ҳ\xc1\x1d\xda%\x99\xb6RQw\x86Qs\x8a\xb8T\xc1T\x19\xba˅U\xf7\xcd\xff\xe2P\x86\xf4\xfa\x04+?\x89̈\xa3q\xed\xd1@\xd6\xfc\x95\x13\x10\xd5\x17\xc1\x94\xa9e\x17\a\xa2\xa5Kع\xfb\xb0\xfe\x02\xe3\xd2\xf90\xa6\xec\x17\xe5\xec'\xd2\xe1\b\x840\xe3\xb6\x18\xcb!f\xe5INtM\xf0\xc6q\xfe\xd0֠\x9b\xd2Oi\xd3\x1b\xa6Q\xccrV5\xdcd\xa7\x81\rB\n\x8dblj\xb8up\xa3z\xb47\x8a\xf0??\x00a\x9a*!\xf6eGpl\x92\xd3\xe0\xc2\xda\xd1\xc0\xe8d\x17\xcekR\xea\xeb\x80ZNO\b\x94\x99fkt.\r\xd8\xfa\b\xeaP\xf9\x03\x81\xf5I\xe6\xf9\xca\xcd\xe0Tl\x91\xa7\xbd\x13,_r\x90,\xffةS\xa3\xf9?\xd6m-^A\x03\x90\xe2\x1e\xdf\xd5g\x19/c\x80Y\xf5\xce\"\x19E,4\b\xafb\x05bRǘΗ\x96\x86.\xf5\xf3\vT\xf0s\xc6\xfcѷW\xc7o\xbcc\x91\xfbՠ{oS\x8fk\xa7\x02u\xfe\x99\xd8[\xc6\xfe\xb7\x80\xb1\\\xa5WC\xc7\x1by\u007fK\x9d\aޡx9^\xde\xc5\x10p\x87\x94\xecEd\x87\xa0\x17\xae\xf7\xa2Mܬo\xbf\x85\x9e\v\xe1W\x0f\xe0BI\x8e-_\xbd\xcf\xebK.\xefQ_2\xa5\xdcG\b\xf2\xa4\x89\x0e\x19\xe9`\x8d\x8f\x86\xbbٌ\x00\x8f\x9d\xd1]\x9e\x98\xc5)\xaeK\xe4\xb5\xc9\x1e\xf6\xed\xf0\xa5\xa6Mę\x02\xa9r\xe1\xcct\v\xf8\xb3\xee\vNti\x81jp\x87\x17\xb9\x19+N\xf4\r~\x96\xe3G\xaau\x8a\x11\x1d\x0fY\xf2\xfd>\x9d\xf0RC\x1b]\xe0\xf7\xbb\x8fϸ\xda\xfbCd~\xc1*\xe3\n\x9a\x10\xb1\"\xd3ʫD\xc6\xc4ײߜ\x93Q\xda\xe9+锨\xd9\x13ů\xc1\x94\x82y\x06\xe2\x87}`1_t\xe5b\x9d\xbe\x03sB\xa4\xfch\xd1j\xfa\\\x92\xb6Ah\xd0\"c\x03\x9b\xa7r\x8b<\x11c\u007f\x8e{\xebc\xafx\x05r\xe1Vlfd$ou\xb5\xb1\xb8\x02\x8e\xe9\x92\xcaf7\x1e:E3ex\xb2\xe7\xcf\x123'\x8c}1^U\x06\\\xf4\xfa\n>\xe1\xe3L\xef\xe7\xe85\x12\xe1y\x19]\xdc\xc9l\x11\x9cu\x92\xbc\x8a\x9a#\x96\x86\xc7\xf6\xd0s(\x19\xa55\x06\xc6\xe6\xd3\xf4\x0f\xe6ի\x93_\x92\xfc\xa9\xbdkL\xf9\xf9\x82?\xfe\\\x94\xac\xd8\u070f\u007f\x1a\xd2\xf9o\x00\x00\x00\xff\xff\xdb\xd9+\xab\xf6\r\x00\x00"), diff --git a/design/general-progress-monitoring.md b/design/general-progress-monitoring.md index 39ed891fbf..831f186e83 100644 --- a/design/general-progress-monitoring.md +++ b/design/general-progress-monitoring.md @@ -123,11 +123,11 @@ from a plugin could cause a backup or restore to move to "PartiallyFailed". If deleted (cancelled), the plug-ins will attempt to delete the snapshots and stop the data movement - this may not be possible with all storage systems. -In addition, for backups (but not restores), there will also be two additional phases, -"FinalizingAfterPluginOperations" and "FinalizingAfterPluginOperationsPartiallyFailed", which will -handle any steps required after plugin operations have all completed. Initially, this will just -include adding any required resources to the backup that might have changed during asynchronous -operation execution, although eventually other cleanup actions could be added to this phase. +In addition, for backups (but not restores), there will also be two additional phases, "Finalizing" +and "FinalizingPartiallyFailed", which will handle any steps required after plugin operations have +all completed. Initially, this will just include adding any required resources to the backup that +might have changed during asynchronous operation execution, although eventually other cleanup +actions could be added to this phase. ### State progression @@ -156,24 +156,24 @@ asynchronous plugin operations and no errors so far, "WaitingForPluginOperations backups or restores which have unfinished asynchronous plugin operations at least one error, "Completed" for restores with no unfinished asynchronous plugin operations and no errors, "PartiallyFailed" for restores with no unfinished asynchronous plugin operations and at least one -error, "FinalizingAfterPluginOperations" for backups with no unfinished asynchronous plugin -operations and no errors, "FinalizingAfterPluginOperationsPartiallyFailed" for backups with no -unfinished asynchronous plugin operations and at least one error, or "PartiallyFailed". -Backups/restores which would have a final phase of "Completed" or "PartiallyFailed" may move to the -"WaitingForPluginOperations" or "WaitingForPluginOperationsPartiallyFailed" state. A backup/restore -which will be marked "Failed" will go directly to the "Failed" phase. Uploads may continue in the -background for snapshots that were taken by a "Failed" backup/restore, but no progress will not be -monitored or updated. If there are any operations in progress when a backup is moved to the "Failed" -phase (although with the current workflow, that shouldn't happen), Cancel() should be called on -these operations. When a "Failed" backup is deleted, all snapshots will be deleted and at that point -any uploads still in progress should be aborted. +error, "Finalizing" for backups with no unfinished asynchronous plugin operations and no errors, +"FinalizingPartiallyFailed" for backups with no unfinished asynchronous plugin operations and at +least one error, or "PartiallyFailed". Backups/restores which would have a final phase of +"Completed" or "PartiallyFailed" may move to the "WaitingForPluginOperations" or +"WaitingForPluginOperationsPartiallyFailed" state. A backup/restore which will be marked "Failed" +will go directly to the "Failed" phase. Uploads may continue in the background for snapshots that +were taken by a "Failed" backup/restore, but no progress will not be monitored or updated. If there +are any operations in progress when a backup is moved to the "Failed" phase (although with the +current workflow, that shouldn't happen), Cancel() should be called on these operations. When a +"Failed" backup is deleted, all snapshots will be deleted and at that point any uploads still in +progress should be aborted. ### WaitingForPluginOperations (new) The "WaitingForPluginOperations" phase signifies that the main part of the backup/restore, including snapshotting has completed successfully and uploading and any other asynchronous BIA/RIA plugin operations are continuing. In the event of an error during this phase, the phase will change to WaitingForPluginOperationsPartiallyFailed. On success, the phase changes to -"FinalizingAfterPluginOperations" for backups and "Completed" for restores. Backups cannot be +"Finalizing" for backups and "Completed" for restores. Backups cannot be restored from when they are in the WaitingForPluginOperations state. ### WaitingForPluginOperationsPartiallyFailed (new) @@ -182,21 +182,19 @@ backup/restore, including snapshotting has completed, but there were partial fai the main part or during any async operations, including snapshot uploads. Backups cannot be restored from when they are in the WaitingForPluginOperationsPartiallyFailed state. -### FinalizingAfterPluginOperations (new) -The "FinalizingAfterPluginOperations" phase signifies that asynchronous backup operations have all -completed successfully and Velero is currently backing up any resources indicated by asynchronous -plugins as items to back up after operations complete. Once this is done, the phase changes to -Completed. Backups cannot be restored from when they are in the FinalizingAfterPluginOperations -state. +### Finalizing (new) +The "Finalizing" phase signifies that asynchronous backup operations have all completed successfully +and Velero is currently backing up any resources indicated by asynchronous plugins as items to back +up after operations complete. Once this is done, the phase changes to Completed. Backups cannot be +restored from when they are in the Finalizing state. -### FinalizingAfterPluginOperationsPartiallyFailed (new) +### FinalizingPartiallyFailed (new) -The "FinalizingAfterPluginOperationsPartiallyFailed" phase signifies that, for a backup which had -errors during initial processing or asynchronous plugin operation, asynchronous backup operations -have all completed and Velero is currently backing up any resources indicated by asynchronous -plugins as items to back up after operations complete. Once this is done, the phase changes to -PartiallyFailed. Backups cannot be restored from when they are in the -FinalizingAfterPluginOperationsPartiallyFailed state. +The "FinalizingPartiallyFailed" phase signifies that, for a backup which had errors during initial +processing or asynchronous plugin operation, asynchronous backup operations have all completed and +Velero is currently backing up any resources indicated by asynchronous plugins as items to back up +after operations complete. Once this is done, the phase changes to PartiallyFailed. Backups cannot +be restored from when they are in the FinalizingPartiallyFailed state. ### Failed When a backup/restore has had fatal errors it is marked as "Failed" Backups in this state cannot be @@ -244,14 +242,14 @@ WaitingForPluginOperationsPartiallyFailed phase, another backup/restore may be s While in the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase, the snapshots and item actions will be periodically polled. When all of the snapshots and item actions have reported success, restores will move directly to the Completed or PartiallyFailed phase, and -backups will move to the FinalizingAfterPluginOperations or -FinalizingAfterPluginOperationsPartiallyFailed phase, depending on whether the backup/restore was in -the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase. +backups will move to the Finalizing or FinalizingPartiallyFailed phase, depending on whether the +backup/restore was in the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed +phase. -While in the FinalizingAfterPluginOperations or FinalizingAfterPluginOperationsPartiallyFailed -phase, Velero will update the backup with any resources indicated by plugins that they must be added -to the backup after operations are completed, and then the backup will move to the Completed or -PartiallyFailed phase, depending on whether there are any backup errors. +While in the Finalizing or FinalizingPartiallyFailed phase, Velero will update the backup with any +resources indicated by plugins that they must be added to the backup after operations are completed, +and then the backup will move to the Completed or PartiallyFailed phase, depending on whether there +are any backup errors. The Backup resources will be written to object storage at the time the backup leaves the InProgress phase, but it will not be synced to other clusters (or usable for restores in the current cluster) @@ -272,7 +270,7 @@ ignored. type OperationProgress struct { Completed bool // True when the operation has completed, either successfully or with a failure Err string // Set when the operation has failed - NCompleted, NTotal int64 // Quantity completed so far and the total quantity associated with the operaation in operationUnits + NCompleted, NTotal int64 // Quantity completed so far and the total quantity associated with the operation in operationUnits // For data mover and volume snapshotter use cases, this would be in bytes // On successful completion, completed and total should be the same. OperationUnits string // Units represented by completed and total -- for data mover and item diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index 9824075b9b..869e374a62 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -226,7 +226,7 @@ const ( // BackupPhase is a string representation of the lifecycle phase // of a Velero backup. -// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;FinalizingAfterPluginOperations;FinalizingAfterPluginOperationsPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting +// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Finalizing;FinalizingPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting type BackupPhase string const ( @@ -256,22 +256,22 @@ const ( // ongoing. The backup is not usable yet. BackupPhaseWaitingForPluginOperationsPartiallyFailed BackupPhase = "WaitingForPluginOperationsPartiallyFailed" - // BackupPhaseFinalizingAfterPluginOperations means the backup of + // BackupPhaseFinalizing means the backup of // Kubernetes resources, creation of snapshots, and other // async plugin operations were successful and snapshot upload and // other plugin operations are now complete, but the Backup is awaiting // final update of resources modified during async operations. // The backup is not usable yet. - BackupPhaseFinalizingAfterPluginOperations BackupPhase = "FinalizingAfterPluginOperations" + BackupPhaseFinalizing BackupPhase = "Finalizing" - // BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed means the backup of + // BackupPhaseFinalizingPartiallyFailed means the backup of // Kubernetes resources, creation of snapshots, and other // async plugin operations were successful and snapshot upload and // other plugin operations are now complete, but one or more errors // occurred during backup or async operation processing, and the // Backup is awaiting final update of resources modified during async // operations. The backup is not usable yet. - BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed BackupPhase = "FinalizingAfterPluginOperationsPartiallyFailed" + BackupPhaseFinalizingPartiallyFailed BackupPhase = "FinalizingPartiallyFailed" // BackupPhaseCompleted means the backup has run successfully without // errors. @@ -374,20 +374,20 @@ type BackupStatus struct { // +optional CSIVolumeSnapshotsCompleted int `json:"csiVolumeSnapshotsCompleted,omitempty"` - // AsyncBackupItemOperationsAttempted is the total number of attempted + // BackupItemOperationsAttempted is the total number of attempted // async BackupItemAction operations for this backup. // +optional - AsyncBackupItemOperationsAttempted int `json:"asyncBackupItemOperationsAttempted,omitempty"` + BackupItemOperationsAttempted int `json:"backupItemOperationsAttempted,omitempty"` - // AsyncBackupItemOperationsCompleted is the total number of successfully completed + // BackupItemOperationsCompleted is the total number of successfully completed // async BackupItemAction operations for this backup. // +optional - AsyncBackupItemOperationsCompleted int `json:"asyncBackupItemOperationsCompleted,omitempty"` + BackupItemOperationsCompleted int `json:"backupItemOperationsCompleted,omitempty"` - // AsyncBackupItemOperationsFailed is the total number of async + // BackupItemOperationsFailed is the total number of async // BackupItemAction operations for this backup which ended with an error. // +optional - AsyncBackupItemOperationsFailed int `json:"asyncBackupItemOperationsFailed,omitempty"` + BackupItemOperationsFailed int `json:"backupItemOperationsFailed,omitempty"` } // BackupProgress stores information about the progress of a Backup's execution. diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 67243361b8..6987142d0f 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -422,7 +422,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger, } func (kb *kubernetesBackupper) backupItem(log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper, unstructured *unstructured.Unstructured, preferredGVR schema.GroupVersionResource) bool { - backedUpItem, err := itemBackupper.backupItem(log, unstructured, gr, preferredGVR, false) + backedUpItem, _, err := itemBackupper.backupItem(log, unstructured, gr, preferredGVR, false, false) if aggregate, ok := err.(kubeerrs.Aggregate); ok { log.WithField("name", unstructured.GetName()).Infof("%d errors encountered backup up item", len(aggregate.Errors())) // log each error separately so we get error location info in the log, and an @@ -441,7 +441,7 @@ func (kb *kubernetesBackupper) backupItem(log logrus.FieldLogger, gr schema.Grou } func (kb *kubernetesBackupper) finalizeItem(log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper, unstructured *unstructured.Unstructured, preferredGVR schema.GroupVersionResource) (bool, []FileForArchive) { - backedUpItem, updateFiles, err := itemBackupper.finalizeItem(log, unstructured, gr, preferredGVR) + backedUpItem, updateFiles, err := itemBackupper.backupItem(log, unstructured, gr, preferredGVR, true, true) if aggregate, ok := err.(kubeerrs.Aggregate); ok { log.WithField("name", unstructured.GetName()).Infof("%d errors encountered backup up item", len(aggregate.Errors())) // log each error separately so we get error location info in the log, and an @@ -563,15 +563,15 @@ func (kb *kubernetesBackupper) FinalizeBackup(log logrus.FieldLogger, pageSize: kb.clientPageSize, } - // Get item list from itemoperation.BackupOperation.Spec.ItemsToUpdate + // Get item list from itemoperation.BackupOperation.Spec.PostOperationItems var resourceIDs []velero.ResourceIdentifier for _, operation := range asyncBIAOperations { - if len(operation.Spec.ItemsToUpdate) != 0 { - resourceIDs = append(resourceIDs, operation.Spec.ItemsToUpdate...) + if len(operation.Spec.PostOperationItems) != 0 { + resourceIDs = append(resourceIDs, operation.Spec.PostOperationItems...) } } items := collector.getItemsFromResourceIdentifiers(resourceIDs) - log.WithField("progress", "").Infof("Collected %d items from the async BIA operations ItemsToUpdate list", len(items)) + log.WithField("progress", "").Infof("Collected %d items from the async BIA operations PostOperationItems list", len(items)) itemBackupper := &itemBackupper{ backupRequest: backupRequest, diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index e67c4e3fc2..bde9a393f8 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -1136,23 +1136,23 @@ func TestBackupResourceOrdering(t *testing.T) { // to run for specific resources/namespaces and simply records the items // that it is executed for. type recordResourcesAction struct { - selector velero.ResourceSelector - ids []string - backups []velerov1.Backup - additionalItems []velero.ResourceIdentifier - operationID string - itemsToUpdate []velero.ResourceIdentifier + selector velero.ResourceSelector + ids []string + backups []velerov1.Backup + additionalItems []velero.ResourceIdentifier + operationID string + postOperationItems []velero.ResourceIdentifier } func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { metadata, err := meta.Accessor(item) if err != nil { - return item, a.additionalItems, a.operationID, a.itemsToUpdate, err + return item, a.additionalItems, a.operationID, a.postOperationItems, err } a.ids = append(a.ids, kubeutil.NamespaceAndName(metadata)) a.backups = append(a.backups, *backup) - return item, a.additionalItems, a.operationID, a.itemsToUpdate, nil + return item, a.additionalItems, a.operationID, a.postOperationItems, nil } func (a *recordResourcesAction) AppliesTo() (velero.ResourceSelector, error) { diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 4bef0f5cf7..98c0cb41ab 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -76,34 +76,27 @@ type FileForArchive struct { FileBytes []byte } -// finalizeItem backs up an individual item and returns its content to replace previous content -// in the backup tarball -// In addition to the error return, backupItem also returns a bool indicating whether the item -// was actually backed up and a slice of filepaths and filecontent to replace the data in the original tarball. -func (ib *itemBackupper) finalizeItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource) (bool, []FileForArchive, error) { - return ib.backupItemInternal(logger, obj, groupResource, preferredGVR, true, true) -} - // backupItem backs up an individual item to tarWriter. The item may be excluded based on the // namespaces IncludesExcludes list. +// If finalize is true, then it returns the bytes instead of writing them to the tarWriter // In addition to the error return, backupItem also returns a bool indicating whether the item // was actually backed up. -func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude bool) (bool, error) { - selectedForBackup, files, err := ib.backupItemInternal(logger, obj, groupResource, preferredGVR, mustInclude, false) - // return if not selected, an error occurred, or there are no files to add - if selectedForBackup == false || err != nil || len(files) == 0 { - return selectedForBackup, err +func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude, finalize bool) (bool, []FileForArchive, error) { + selectedForBackup, files, err := ib.backupItemInternal(logger, obj, groupResource, preferredGVR, mustInclude, finalize) + // return if not selected, an error occurred, there are no files to add, or for finalize + if selectedForBackup == false || err != nil || len(files) == 0 || finalize { + return selectedForBackup, files, err } for _, file := range files { if err := ib.tarWriter.WriteHeader(file.Header); err != nil { - return false, errors.WithStack(err) + return false, []FileForArchive{}, errors.WithStack(err) } if _, err := ib.tarWriter.Write(file.FileBytes); err != nil { - return false, errors.WithStack(err) + return false, []FileForArchive{}, errors.WithStack(err) } } - return true, nil + return true, []FileForArchive{}, nil } func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude, finalize bool) (bool, []FileForArchive, error) { @@ -178,7 +171,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti return false, itemFiles, err } - if !finalize && groupResource == kuberesource.Pods { + if groupResource == kuberesource.Pods { // pod needs to be initialized for the unstructured converter pod = new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pod); err != nil { @@ -211,7 +204,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti // the group version of the object. versionPath := resourceVersion(obj) - updatedObj, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata, finalize) + updatedObj, additionalItemFiles, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata, finalize) if err != nil { backupErrs = append(backupErrs, err) @@ -222,6 +215,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti } return false, itemFiles, kubeerrs.NewAggregate(backupErrs) } + itemFiles = append(itemFiles, additionalItemFiles...) obj = updatedObj if metadata, err = meta.Accessor(obj); err != nil { return false, itemFiles, errors.WithStack(err) @@ -230,13 +224,13 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti name = metadata.GetName() namespace = metadata.GetNamespace() - if !finalize && groupResource == kuberesource.PersistentVolumes { + if groupResource == kuberesource.PersistentVolumes { if err := ib.takePVSnapshot(obj, log); err != nil { backupErrs = append(backupErrs, err) } } - if !finalize && groupResource == kuberesource.Pods && pod != nil { + if groupResource == kuberesource.Pods && pod != nil { // this function will return partial results, so process podVolumeBackups // even if there are errors. podVolumeBackups, errs := ib.backupPodVolumes(log, pod, pvbVolumes) @@ -309,33 +303,34 @@ func (ib *itemBackupper) executeActions( name, namespace string, metadata metav1.Object, finalize bool, -) (runtime.Unstructured, error) { +) (runtime.Unstructured, []FileForArchive, error) { + var itemFiles []FileForArchive for _, action := range ib.backupRequest.ResolvedActions { if !action.ShouldUse(groupResource, namespace, metadata, log) { continue } log.Info("Executing custom action") - updatedItem, additionalItemIdentifiers, operationID, itemsToUpdate, err := action.Execute(obj, ib.backupRequest.Backup) + updatedItem, additionalItemIdentifiers, operationID, postOperationItems, err := action.Execute(obj, ib.backupRequest.Backup) if err != nil { - return nil, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) + return nil, itemFiles, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} - mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true" + mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true" || finalize // remove the annotation as it's for communication between BIA and velero server, // we don't want the resource be restored with this annotation. if _, ok := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation]; ok { delete(u.GetAnnotations(), mustIncludeAdditionalItemAnnotation) } obj = u - if finalize { - continue - } // If async plugin started async operation, add it to the ItemOperations list // ignore during finalize phase if operationID != "" { + if finalize { + return nil, itemFiles, errors.New(fmt.Sprintf("Backup Item Action created operation during finalize (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name)) + } resourceIdentifier := velero.ResourceIdentifier{ GroupResource: groupResource, Namespace: namespace, @@ -355,7 +350,7 @@ func (ib *itemBackupper) executeActions( Created: &now, }, } - newOperation.Spec.ItemsToUpdate = itemsToUpdate + newOperation.Spec.PostOperationItems = postOperationItems itemOperList := ib.backupRequest.GetItemOperationsList() *itemOperList = append(*itemOperList, &newOperation) } @@ -363,12 +358,12 @@ func (ib *itemBackupper) executeActions( for _, additionalItem := range additionalItemIdentifiers { gvr, resource, err := ib.discoveryHelper.ResourceFor(additionalItem.GroupResource.WithVersion("")) if err != nil { - return nil, err + return nil, itemFiles, err } client, err := ib.dynamicFactory.ClientForGroupVersionResource(gvr.GroupVersion(), resource, additionalItem.Namespace) if err != nil { - return nil, err + return nil, itemFiles, err } item, err := client.Get(additionalItem.Name, metav1.GetOptions{}) @@ -382,15 +377,17 @@ func (ib *itemBackupper) executeActions( continue } if err != nil { - return nil, errors.WithStack(err) + return nil, itemFiles, errors.WithStack(err) } - if _, err = ib.backupItem(log, item, gvr.GroupResource(), gvr, mustInclude); err != nil { - return nil, err + _, additionalItemFiles, err := ib.backupItem(log, item, gvr.GroupResource(), gvr, mustInclude, finalize) + if err != nil { + return nil, itemFiles, err } + itemFiles = append(itemFiles, additionalItemFiles...) } } - return obj, nil + return obj, itemFiles, nil } // volumeSnapshotter instantiates and initializes a VolumeSnapshotter given a VolumeSnapshotLocation, diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index a176e7c22a..0b21359f30 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -75,7 +75,14 @@ func (r *itemCollector) getAllItems() []*kubernetesResource { return r.getItems(nil) } -// getAllItems gets all relevant items from all API groups. +// getItems gets all relevant items from all API groups. +// If resourceIDsMap is nil, then all items from the cluster are +// pulled for each API group, subject to include/exclude rules. +// If resourceIDsMap is supplied, then only those resources are +// returned, with the appropriate APIGroup information filled in. In +// this case, include/exclude rules are not invoked, since we already +// have the list of items, we just need the item collector/discovery +// helper to fill in the missing GVR, etc. context. func (r *itemCollector) getItems(resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier) []*kubernetesResource { var resources []*kubernetesResource for _, group := range r.discoveryHelper.Resources() { @@ -92,6 +99,8 @@ func (r *itemCollector) getItems(resourceIDsMap map[schema.GroupResource][]veler } // getGroupItems collects all relevant items from a single API group. +// If resourceIDsMap is supplied, then only those items are returned, +// with GVR/APIResource metadata supplied. func (r *itemCollector) getGroupItems(log logrus.FieldLogger, group *metav1.APIResourceList, resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier) ([]*kubernetesResource, error) { log = log.WithField("group", group.GroupVersion) @@ -181,6 +190,8 @@ func getOrderedResourcesForType(orderedResources map[string]string, resourceType } // getResourceItems collects all relevant items for a given group-version-resource. +// If resourceIDsMap is supplied, the items will be pulled from here +// rather than from the cluster and applying include/exclude rules. func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.GroupVersion, resource metav1.APIResource, resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier) ([]*kubernetesResource, error) { log = log.WithField("resource", resource.Name) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 91d03b5297..d1d2e13e90 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -65,6 +65,7 @@ import ( velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" + "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/persistence" @@ -226,7 +227,7 @@ func NewCommand(f client.Factory) *cobra.Command { command.Flags().DurationVar(&config.defaultBackupTTL, "default-backup-ttl", config.defaultBackupTTL, "How long to wait by default before backups can be garbage collected.") command.Flags().DurationVar(&config.repoMaintenanceFrequency, "default-repo-maintain-frequency", config.repoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default.") command.Flags().DurationVar(&config.garbageCollectionFrequency, "garbage-collection-frequency", config.garbageCollectionFrequency, "How often garbage collection is run for expired backups.") - command.Flags().DurationVar(&config.itemOperationSyncFrequency, "item-operation-sync-frequency", config.itemOperationSyncFrequency, "How often to check status on async backup/restore operations after backup processing.") + command.Flags().DurationVar(&config.itemOperationSyncFrequency, "item-operation-sync-frequency", config.itemOperationSyncFrequency, "How often to check status on backup/restore operations after backup/restore processing.") command.Flags().BoolVar(&config.defaultVolumesToFsBackup, "default-volumes-to-fs-backup", config.defaultVolumesToFsBackup, "Backup all volumes with pod volume file system backup by default.") command.Flags().StringVar(&config.uploaderType, "uploader-type", config.uploaderType, "Type of uploader to handle the transfer of data of pod volumes") command.Flags().DurationVar(&config.defaultItemOperationTimeout, "default-item-operation-timeout", config.defaultItemOperationTimeout, "How long to wait on asynchronous BackupItemActions and RestoreItemActions to complete before timing out.") @@ -642,26 +643,26 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string // and BSL controller is mandatory for Velero to work. // Note: all runtime type controllers that can be disabled are grouped separately, below: enabledRuntimeControllers := map[string]struct{}{ - controller.AsyncBackupOperations: {}, - controller.Backup: {}, - controller.BackupDeletion: {}, - controller.BackupFinalizer: {}, - controller.BackupRepo: {}, - controller.BackupSync: {}, - controller.DownloadRequest: {}, - controller.GarbageCollection: {}, - controller.Restore: {}, - controller.Schedule: {}, - controller.ServerStatusRequest: {}, + controller.Backup: {}, + controller.BackupDeletion: {}, + controller.BackupFinalizer: {}, + controller.BackupOperations: {}, + controller.BackupRepo: {}, + controller.BackupSync: {}, + controller.DownloadRequest: {}, + controller.GarbageCollection: {}, + controller.Restore: {}, + controller.Schedule: {}, + controller.ServerStatusRequest: {}, } if s.config.restoreOnly { s.logger.Info("Restore only mode - not starting the backup, schedule, delete-backup, or GC controllers") s.config.disabledControllers = append(s.config.disabledControllers, - controller.AsyncBackupOperations, controller.Backup, controller.BackupDeletion, controller.BackupFinalizer, + controller.BackupOperations, controller.GarbageCollection, controller.Schedule, ) @@ -690,22 +691,6 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.logger.Fatal(err, "unable to create controller", "controller", controller.BackupStorageLocation) } - var backupOpsMap *controller.BackupItemOperationsMap - if _, ok := enabledRuntimeControllers[controller.AsyncBackupOperations]; ok { - r, m := controller.NewAsyncBackupOperationsReconciler( - s.logger, - s.mgr.GetClient(), - s.config.itemOperationSyncFrequency, - newPluginManager, - backupStoreGetter, - s.metrics, - ) - if err := r.SetupWithManager(s.mgr); err != nil { - s.logger.Fatal(err, "unable to create controller", "controller", controller.AsyncBackupOperations) - } - backupOpsMap = m - } - if _, ok := enabledRuntimeControllers[controller.Backup]; ok { backupper, err := backup.NewKubernetesBackupper( s.mgr.GetClient(), @@ -770,6 +755,22 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + backupOpsMap := itemoperationmap.NewBackupItemOperationsMap() + if _, ok := enabledRuntimeControllers[controller.BackupOperations]; ok { + r := controller.NewBackupOperationsReconciler( + s.logger, + s.mgr.GetClient(), + s.config.itemOperationSyncFrequency, + newPluginManager, + backupStoreGetter, + s.metrics, + backupOpsMap, + ) + if err := r.SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", controller.BackupOperations) + } + } + if _, ok := enabledRuntimeControllers[controller.BackupFinalizer]; ok { backupper, err := backup.NewKubernetesBackupper( s.mgr.GetClient(), diff --git a/pkg/cmd/server/server_test.go b/pkg/cmd/server/server_test.go index 52d7de106e..bab98b42fe 100644 --- a/pkg/cmd/server/server_test.go +++ b/pkg/cmd/server/server_test.go @@ -89,7 +89,7 @@ func TestRemoveControllers(t *testing.T) { { name: "Remove all disable controllers", disabledControllers: []string{ - controller.AsyncBackupOperations, + controller.BackupOperations, controller.Backup, controller.BackupDeletion, controller.BackupSync, @@ -121,16 +121,16 @@ func TestRemoveControllers(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { enabledRuntimeControllers := map[string]struct{}{ - controller.BackupSync: {}, - controller.Backup: {}, - controller.GarbageCollection: {}, - controller.Restore: {}, - controller.ServerStatusRequest: {}, - controller.Schedule: {}, - controller.BackupDeletion: {}, - controller.BackupRepo: {}, - controller.DownloadRequest: {}, - controller.AsyncBackupOperations: {}, + controller.BackupSync: {}, + controller.Backup: {}, + controller.GarbageCollection: {}, + controller.Restore: {}, + controller.ServerStatusRequest: {}, + controller.Schedule: {}, + controller.BackupDeletion: {}, + controller.BackupRepo: {}, + controller.DownloadRequest: {}, + controller.BackupOperations: {}, } totalNumOriginalControllers := len(enabledRuntimeControllers) diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 8c73e6cf7f..4c2e9d271c 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -68,7 +68,7 @@ func DescribeBackup( phaseString = color.GreenString(phaseString) case velerov1api.BackupPhaseDeleting: case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: - case velerov1api.BackupPhaseFinalizingAfterPluginOperations, velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed: + case velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: case velerov1api.BackupPhaseInProgress: case velerov1api.BackupPhaseNew: } @@ -288,7 +288,7 @@ func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Desc d.Println() } - describeAsyncBackupItemOperations(ctx, kbClient, d, backup, details, insecureSkipTLSVerify, caCertPath) + describeBackupItemOperations(ctx, kbClient, d, backup, details, insecureSkipTLSVerify, caCertPath) if details { describeBackupResourceList(ctx, kbClient, d, backup, insecureSkipTLSVerify, caCertPath) @@ -323,29 +323,29 @@ func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Desc d.Printf("Velero-Native Snapshots: \n") } -func describeAsyncBackupItemOperations(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string) { +func describeBackupItemOperations(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string) { status := backup.Status - if status.AsyncBackupItemOperationsAttempted > 0 { + if status.BackupItemOperationsAttempted > 0 { if !details { - d.Printf("Async Backup Item Operations:\t%d of %d completed successfully, %d failed (specify --details for more information)\n", status.AsyncBackupItemOperationsCompleted, status.AsyncBackupItemOperationsAttempted, status.AsyncBackupItemOperationsFailed) + d.Printf("Backup Item Operations:\t%d of %d completed successfully, %d failed (specify --details for more information)\n", status.BackupItemOperationsCompleted, status.BackupItemOperationsAttempted, status.BackupItemOperationsFailed) return } buf := new(bytes.Buffer) if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { - d.Printf("Async Backup Item Operations:\t\n", err) + d.Printf("Backup Item Operations:\t\n", err) return } var operations []*itemoperation.BackupOperation if err := json.NewDecoder(buf).Decode(&operations); err != nil { - d.Printf("Async Backup Item Operations:\t\n", err) + d.Printf("Backup Item Operations:\t\n", err) return } - d.Printf("Async Backup Item Operations:\n") + d.Printf("Backup Item Operations:\n") for _, operation := range operations { - describeAsyncBackupItemOperation(d, operation) + describeBackupItemOperation(d, operation) } } } @@ -398,14 +398,14 @@ func describeSnapshot(d *Describer, pvName, snapshotID, volumeType, volumeAZ str d.Printf("\t\tIOPS:\t%s\n", iopsString) } -func describeAsyncBackupItemOperation(d *Describer, operation *itemoperation.BackupOperation) { +func describeBackupItemOperation(d *Describer, operation *itemoperation.BackupOperation) { d.Printf("\tOperation for %s %s/%s:\n", operation.Spec.ResourceIdentifier, operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) d.Printf("\t\tBackup Item Action Plugin:\t%s\n", operation.Spec.BackupItemAction) d.Printf("\t\tOperation ID:\t%s\n", operation.Spec.OperationID) - if len(operation.Spec.ItemsToUpdate) > 0 { + if len(operation.Spec.PostOperationItems) > 0 { d.Printf("\t\tItems to Update:\n") } - for _, item := range operation.Spec.ItemsToUpdate { + for _, item := range operation.Spec.PostOperationItems { d.Printf("\t\t\t%s %s/%s\n", item, item.Namespace, item.Name) } d.Printf("\t\tPhase:\t%s\n", operation.Status.Phase) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 48180c78b8..e6a09f1f50 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -18,9 +18,7 @@ package controller import ( "bytes" - "compress/gzip" "context" - "encoding/json" "fmt" "io/ioutil" "os" @@ -687,9 +685,9 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error { } } - backup.Status.AsyncBackupItemOperationsAttempted = len(*backup.GetItemOperationsList()) - backup.Status.AsyncBackupItemOperationsCompleted = opsCompleted - backup.Status.AsyncBackupItemOperationsFailed = opsFailed + backup.Status.BackupItemOperationsAttempted = len(*backup.GetItemOperationsList()) + backup.Status.BackupItemOperationsCompleted = opsCompleted + backup.Status.BackupItemOperationsFailed = opsFailed backup.Status.Warnings = logCounter.GetCount(logrus.WarnLevel) backup.Status.Errors = logCounter.GetCount(logrus.ErrorLevel) @@ -714,13 +712,13 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error { if inProgressOperations { backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed } else { - backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed + backup.Status.Phase = velerov1api.BackupPhaseFinalizingPartiallyFailed } default: if inProgressOperations { backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperations } else { - backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperations + backup.Status.Phase = velerov1api.BackupPhaseFinalizing } } // Mark completion timestamp before serializing and uploading. @@ -811,42 +809,42 @@ func persistBackup(backup *pkgbackup.Request, } // Velero-native volume snapshots (as opposed to CSI ones) - nativeVolumeSnapshots, errs := encodeToJSONGzip(backup.VolumeSnapshots, "native volumesnapshots list") + nativeVolumeSnapshots, errs := encode.EncodeToJSONGzip(backup.VolumeSnapshots, "native volumesnapshots list") if errs != nil { persistErrs = append(persistErrs, errs...) } var backupItemOperations *bytes.Buffer - backupItemOperations, errs = encodeToJSONGzip(backup.GetItemOperationsList(), "backup item operations list") + backupItemOperations, errs = encode.EncodeToJSONGzip(backup.GetItemOperationsList(), "backup item operations list") if errs != nil { persistErrs = append(persistErrs, errs...) } - podVolumeBackups, errs := encodeToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") + podVolumeBackups, errs := encode.EncodeToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") if errs != nil { persistErrs = append(persistErrs, errs...) } - csiSnapshotJSON, errs := encodeToJSONGzip(csiVolumeSnapshots, "csi volume snapshots list") + csiSnapshotJSON, errs := encode.EncodeToJSONGzip(csiVolumeSnapshots, "csi volume snapshots list") if errs != nil { persistErrs = append(persistErrs, errs...) } - csiSnapshotContentsJSON, errs := encodeToJSONGzip(csiVolumeSnapshotContents, "csi volume snapshot contents list") + csiSnapshotContentsJSON, errs := encode.EncodeToJSONGzip(csiVolumeSnapshotContents, "csi volume snapshot contents list") if errs != nil { persistErrs = append(persistErrs, errs...) } - csiSnapshotClassesJSON, errs := encodeToJSONGzip(csiVolumesnapshotClasses, "csi volume snapshot classes list") + csiSnapshotClassesJSON, errs := encode.EncodeToJSONGzip(csiVolumesnapshotClasses, "csi volume snapshot classes list") if errs != nil { persistErrs = append(persistErrs, errs...) } - backupResourceList, errs := encodeToJSONGzip(backup.BackupResourceList(), "backup resources list") + backupResourceList, errs := encode.EncodeToJSONGzip(backup.BackupResourceList(), "backup resources list") if errs != nil { persistErrs = append(persistErrs, errs...) } - backupResult, errs := encodeToJSONGzip(results, "backup results") + backupResult, errs := encode.EncodeToJSONGzip(results, "backup results") if errs != nil { persistErrs = append(persistErrs, errs...) } @@ -898,29 +896,6 @@ func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { } } -// encodeToJSONGzip takes arbitrary Go data and encodes it to GZip compressed JSON in a buffer, as well as a description of the data to put into an error should encoding fail. -func encodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, []error) { - buf := new(bytes.Buffer) - gzw := gzip.NewWriter(buf) - - // Since both encoding and closing the gzip writer could fail separately and both errors are useful, - // collect both errors to report back. - errs := []error{} - - if err := json.NewEncoder(gzw).Encode(data); err != nil { - errs = append(errs, errors.Wrapf(err, "error encoding %s", desc)) - } - if err := gzw.Close(); err != nil { - errs = append(errs, errors.Wrapf(err, "error closing gzip writer for %s", desc)) - } - - if len(errs) > 0 { - return nil, errs - } - - return buf, nil -} - // waitVolumeSnapshotReadyToUse is used to wait VolumeSnapshot turned to ReadyToUse. // Waiting for VolumeSnapshot ReadyToUse to true is time consuming. Try to make the process parallel by // using goroutine here instead of waiting in CSI plugin, because it's not easy to make BackupItemAction diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 9b31d25173..566add5f23 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -580,7 +580,7 @@ func TestProcessBackupCompletions(t *testing.T) { backupExists bool existenceCheckError error }{ - // FinalizingAfterPluginOperations + // Finalizing { name: "backup with no backup location gets the default", backup: defaultBackup().Result(), @@ -608,7 +608,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -643,7 +643,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -681,7 +681,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -717,7 +717,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", Expiration: &metav1.Time{now.Add(10 * time.Minute)}, @@ -753,7 +753,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -790,7 +790,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -827,7 +827,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -864,7 +864,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -901,7 +901,7 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, diff --git a/pkg/controller/backup_finalizer_controller.go b/pkg/controller/backup_finalizer_controller.go index 562efc965e..25de5f660c 100644 --- a/pkg/controller/backup_finalizer_controller.go +++ b/pkg/controller/backup_finalizer_controller.go @@ -94,7 +94,7 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ } switch backup.Status.Phase { - case velerov1api.BackupPhaseFinalizingAfterPluginOperations, velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed: + case velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: // only process backups finalizing after plugin operations are complete default: log.Debug("Backup is not awaiting finalizing, skipping") @@ -168,11 +168,11 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ } backupScheduleName := backupRequest.GetLabels()[velerov1api.ScheduleNameLabel] switch backup.Status.Phase { - case velerov1api.BackupPhaseFinalizingAfterPluginOperations: + case velerov1api.BackupPhaseFinalizing: backup.Status.Phase = velerov1api.BackupPhaseCompleted r.metrics.RegisterBackupSuccess(backupScheduleName) r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusSucc) - case velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed: + case velerov1api.BackupPhaseFinalizingPartiallyFailed: backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed r.metrics.RegisterBackupPartialFailure(backupScheduleName) r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure) diff --git a/pkg/controller/backup_finalizer_controller_test.go b/pkg/controller/backup_finalizer_controller_test.go index 828412a7f9..e062e8a722 100644 --- a/pkg/controller/backup_finalizer_controller_test.go +++ b/pkg/controller/backup_finalizer_controller_test.go @@ -72,12 +72,12 @@ func TestBackupFinalizerReconcile(t *testing.T) { expectPhase velerov1api.BackupPhase }{ { - name: "FinalizingAfterPluginOperations backup is completed", + name: "Finalizing backup is completed", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). StorageLocation("default"). ObjectMeta(builder.WithUID("foo")). StartTimestamp(fakeClock.Now()). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + Phase(velerov1api.BackupPhaseFinalizing).Result(), backupLocation: defaultBackupLocation, expectPhase: velerov1api.BackupPhaseCompleted, backupOperations: []*itemoperation.BackupOperation{ @@ -91,7 +91,7 @@ func TestBackupFinalizerReconcile(t *testing.T) { Namespace: "ns-1", Name: "pod-1", }, - ItemsToUpdate: []velero.ResourceIdentifier{ + PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "ns-1", @@ -108,12 +108,12 @@ func TestBackupFinalizerReconcile(t *testing.T) { }, }, { - name: "FinalizingAfterPluginOperationsPartiallyFailed backup is partially failed", + name: "FinalizingPartiallyFailed backup is partially failed", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). StorageLocation("default"). ObjectMeta(builder.WithUID("foo")). StartTimestamp(fakeClock.Now()). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed).Result(), + Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed).Result(), backupLocation: defaultBackupLocation, expectPhase: velerov1api.BackupPhasePartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ @@ -127,7 +127,7 @@ func TestBackupFinalizerReconcile(t *testing.T) { Namespace: "ns-2", Name: "pod-2", }, - ItemsToUpdate: []velero.ResourceIdentifier{ + PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "ns-2", diff --git a/pkg/controller/async_backup_operations_controller.go b/pkg/controller/backup_operations_controller.go similarity index 59% rename from pkg/controller/async_backup_operations_controller.go rename to pkg/controller/backup_operations_controller.go index 1d46ecb6f7..cae7c80d53 100644 --- a/pkg/controller/async_backup_operations_controller.go +++ b/pkg/controller/backup_operations_controller.go @@ -19,7 +19,6 @@ package controller import ( "bytes" "context" - "sync" "time" "github.com/pkg/errors" @@ -33,6 +32,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" @@ -41,129 +41,46 @@ import ( ) const ( - defaultAsyncBackupOperationsFrequency = 2 * time.Minute + defaultBackupOperationsFrequency = 2 * time.Minute ) -type operationsForBackup struct { - operations []*itemoperation.BackupOperation - changesSinceUpdate bool - errsSinceUpdate []string -} - -// FIXME: remove if handled by backup finalizer controller -func (o *operationsForBackup) anyItemsToUpdate() bool { - for _, op := range o.operations { - if len(op.Spec.ItemsToUpdate) > 0 { - return true - } - } - return false -} -func (in *operationsForBackup) DeepCopy() *operationsForBackup { - if in == nil { - return nil - } - out := new(operationsForBackup) - in.DeepCopyInto(out) - return out -} - -func (in *operationsForBackup) DeepCopyInto(out *operationsForBackup) { - *out = *in - if in.operations != nil { - in, out := &in.operations, &out.operations - *out = make([]*itemoperation.BackupOperation, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = new(itemoperation.BackupOperation) - (*in).DeepCopyInto(*out) - } - } - } - if in.errsSinceUpdate != nil { - in, out := &in.errsSinceUpdate, &out.errsSinceUpdate - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -func (o *operationsForBackup) uploadProgress(backupStore persistence.BackupStore, backupName string) error { - if len(o.operations) > 0 { - var backupItemOperations *bytes.Buffer - backupItemOperations, errs := encodeToJSONGzip(o.operations, "backup item operations list") - if errs != nil { - return errors.Wrap(errs[0], "error encoding item operations json") - } - err := backupStore.PutBackupItemOperations(backupName, backupItemOperations) - if err != nil { - return errors.Wrap(err, "error uploading item operations json") - } - } - o.changesSinceUpdate = false - o.errsSinceUpdate = nil - return nil -} - -type BackupItemOperationsMap struct { - operations map[string]*operationsForBackup - opsLock sync.Mutex -} - -// If backup has changes not yet uploaded, upload them now -func (m *BackupItemOperationsMap) UpdateForBackup(backupStore persistence.BackupStore, backupName string) error { - // lock operations map - m.opsLock.Lock() - defer m.opsLock.Unlock() - - operations, ok := m.operations[backupName] - // if operations for this backup aren't found, or if there are no changes - // or errors since last update, do nothing - if !ok || (!operations.changesSinceUpdate && len(operations.errsSinceUpdate) == 0) { - return nil - } - if err := operations.uploadProgress(backupStore, backupName); err != nil { - return err - } - return nil -} - -type asyncBackupOperationsReconciler struct { +type backupOperationsReconciler struct { client.Client logger logrus.FieldLogger clock clocks.WithTickerAndDelayedExecution frequency time.Duration - itemOperationsMap *BackupItemOperationsMap + itemOperationsMap *itemoperationmap.BackupItemOperationsMap newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter metrics *metrics.ServerMetrics } -func NewAsyncBackupOperationsReconciler( +func NewBackupOperationsReconciler( logger logrus.FieldLogger, client client.Client, frequency time.Duration, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, metrics *metrics.ServerMetrics, -) (*asyncBackupOperationsReconciler, *BackupItemOperationsMap) { - abor := &asyncBackupOperationsReconciler{ + itemOperationsMap *itemoperationmap.BackupItemOperationsMap, +) *backupOperationsReconciler { + abor := &backupOperationsReconciler{ Client: client, logger: logger, clock: clocks.RealClock{}, frequency: frequency, - itemOperationsMap: &BackupItemOperationsMap{operations: make(map[string]*operationsForBackup)}, + itemOperationsMap: itemOperationsMap, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, metrics: metrics, } if abor.frequency <= 0 { - abor.frequency = defaultAsyncBackupOperationsFrequency + abor.frequency = defaultBackupOperationsFrequency } - return abor, abor.itemOperationsMap + return abor } -func (c *asyncBackupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (c *backupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error { s := kube.NewPeriodicalEnqueueSource(c.logger, mgr.GetClient(), &velerov1api.BackupList{}, c.frequency, kube.PeriodicalEnqueueSourceOption{}) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}, builder.WithPredicates(kube.FalsePredicate{})). @@ -175,10 +92,10 @@ func (c *asyncBackupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) err // +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get // +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get -func (c *asyncBackupOperationsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := c.logger.WithField("async backup operations for backup", req.String()) +func (c *backupOperationsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := c.logger.WithField("backup operations for backup", req.String()) // FIXME: make this log.Debug - log.Info("asyncBackupOperationsReconciler getting backup") + log.Info("backupOperationsReconciler getting backup") original := &velerov1api.Backup{} if err := c.Get(ctx, req.NamespacedName, original); err != nil { @@ -201,7 +118,7 @@ func (c *asyncBackupOperationsReconciler) Reconcile(ctx context.Context, req ctr case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: // only process backups waiting for plugin operations to complete default: - log.Debug("Backup has no ongoing async plugin operations, skipping") + log.Debug("Backup has no ongoing plugin operations, skipping") return ctrl.Result{}, nil } @@ -211,13 +128,13 @@ func (c *asyncBackupOperationsReconciler) Reconcile(ctx context.Context, req ctr Name: backup.Spec.StorageLocation, }, loc); err != nil { if apierrors.IsNotFound(err) { - log.Warnf("Cannot check progress on async Backup operations because backup storage location %s does not exist; marking backup PartiallyFailed", backup.Spec.StorageLocation) + log.Warnf("Cannot check progress on Backup operations because backup storage location %s does not exist; marking backup PartiallyFailed", backup.Spec.StorageLocation) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed } else { - log.Warnf("Cannot check progress on async Backup operations because backup storage location %s could not be retrieved: %s; marking backup PartiallyFailed", backup.Spec.StorageLocation, err.Error()) + log.Warnf("Cannot check progress on Backup operations because backup storage location %s could not be retrieved: %s; marking backup PartiallyFailed", backup.Spec.StorageLocation, err.Error()) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed } - err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &operationsForBackup{errsSinceUpdate: []string{err.Error()}}, false, false) + err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &itemoperationmap.OperationsForBackup{ErrsSinceUpdate: []string{err.Error()}}, false, false) if err2 != nil { log.WithError(err2).Error("error updating Backup") } @@ -225,10 +142,10 @@ func (c *asyncBackupOperationsReconciler) Reconcile(ctx context.Context, req ctr } if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { - log.Infof("Cannot check progress on async Backup operations because backup storage location %s is currently in read-only mode; marking backup PartiallyFailed", loc.Name) + log.Infof("Cannot check progress on Backup operations because backup storage location %s is currently in read-only mode; marking backup PartiallyFailed", loc.Name) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed - err := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &operationsForBackup{errsSinceUpdate: []string{"BSL is read-only"}}, false, false) + err := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &itemoperationmap.OperationsForBackup{ErrsSinceUpdate: []string{"BSL is read-only"}}, false, false) if err != nil { log.WithError(err).Error("error updating Backup") } @@ -242,87 +159,85 @@ func (c *asyncBackupOperationsReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, errors.Wrap(err, "error getting backup store") } - operations, err := c.getOperationsForBackup(backupStore, backup.Name) + operations, err := c.itemOperationsMap.GetOperationsForBackup(backupStore, backup.Name) if err != nil { - err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, &operationsForBackup{errsSinceUpdate: []string{err.Error()}}, false, false) + err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, &itemoperationmap.OperationsForBackup{ErrsSinceUpdate: []string{err.Error()}}, false, false) if err2 != nil { return ctrl.Result{}, errors.Wrap(err2, "error updating Backup") } return ctrl.Result{}, errors.Wrap(err, "error getting backup operations") } - stillInProgress, changes, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup, pluginManager, operations.operations) + stillInProgress, changes, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup, pluginManager, operations.Operations) // if len(errs)>0, need to update backup errors and error log - operations.errsSinceUpdate = append(operations.errsSinceUpdate, errs...) - backup.Status.Errors += len(operations.errsSinceUpdate) - asyncCompletionChanges := false - if backup.Status.AsyncBackupItemOperationsCompleted != opsCompleted || backup.Status.AsyncBackupItemOperationsFailed != opsFailed { - asyncCompletionChanges = true - backup.Status.AsyncBackupItemOperationsCompleted = opsCompleted - backup.Status.AsyncBackupItemOperationsFailed = opsFailed + operations.ErrsSinceUpdate = append(operations.ErrsSinceUpdate, errs...) + backup.Status.Errors += len(operations.ErrsSinceUpdate) + completionChanges := false + if backup.Status.BackupItemOperationsCompleted != opsCompleted || backup.Status.BackupItemOperationsFailed != opsFailed { + completionChanges = true + backup.Status.BackupItemOperationsCompleted = opsCompleted + backup.Status.BackupItemOperationsFailed = opsFailed } if changes { - operations.changesSinceUpdate = true + operations.ChangesSinceUpdate = true } // if stillInProgress is false, backup moves to finalize phase and needs update - // if operations.errsSinceUpdate is not empty, then backup phase needs to change to + // if operations.ErrsSinceUpdate is not empty, then backup phase needs to change to // BackupPhaseWaitingForPluginOperationsPartiallyFailed and needs update // If the only changes are incremental progress, then no write is necessary, progress can remain in memory if !stillInProgress { - if len(operations.errsSinceUpdate) > 0 { + if len(operations.ErrsSinceUpdate) > 0 { backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed } if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations { - log.Infof("Marking backup %s FinalizingAfterPluginOperations", backup.Name) - backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperations + log.Infof("Marking backup %s Finalizing", backup.Name) + backup.Status.Phase = velerov1api.BackupPhaseFinalizing } else { - log.Infof("Marking backup %s FinalizingAfterPluginOperationsPartiallyFailed", backup.Name) - backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed + log.Infof("Marking backup %s FinalizingPartiallyFailed", backup.Name) + backup.Status.Phase = velerov1api.BackupPhaseFinalizingPartiallyFailed } } - err = c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, operations, asyncCompletionChanges, changes) + err = c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, operations, changes, completionChanges) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error updating Backup") } return ctrl.Result{}, nil } -func (c *asyncBackupOperationsReconciler) updateBackupAndOperationsJSON( +func (c *backupOperationsReconciler) updateBackupAndOperationsJSON( ctx context.Context, original, backup *velerov1api.Backup, backupStore persistence.BackupStore, - operations *operationsForBackup, + operations *itemoperationmap.OperationsForBackup, changes bool, - asyncCompletionChanges bool) error { + completionChanges bool) error { backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel] - if len(operations.errsSinceUpdate) > 0 { + if len(operations.ErrsSinceUpdate) > 0 { c.metrics.RegisterBackupItemsErrorsGauge(backupScheduleName, backup.Status.Errors) // FIXME: download/upload results once https://github.com/vmware-tanzu/velero/pull/5576 is merged } removeIfComplete := true defer func() { // remove local operations list if complete - c.itemOperationsMap.opsLock.Lock() if removeIfComplete && (backup.Status.Phase == velerov1api.BackupPhaseCompleted || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed) { + backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed) { - c.deleteOperationsForBackup(backup.Name) + c.itemOperationsMap.DeleteOperationsForBackup(backup.Name) } else if changes { - c.putOperationsForBackup(operations, backup.Name) + c.itemOperationsMap.PutOperationsForBackup(operations, backup.Name) } - c.itemOperationsMap.opsLock.Unlock() }() // update backup and upload progress if errs or complete - if len(operations.errsSinceUpdate) > 0 || + if len(operations.ErrsSinceUpdate) > 0 || backup.Status.Phase == velerov1api.BackupPhaseCompleted || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed { + backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { // update file store if backupStore != nil { backupJSON := new(bytes.Buffer) @@ -335,7 +250,7 @@ func (c *asyncBackupOperationsReconciler) updateBackupAndOperationsJSON( removeIfComplete = false return errors.Wrap(err, "error uploading backup json") } - if err := operations.uploadProgress(backupStore, backup.Name); err != nil { + if err := c.itemOperationsMap.UploadProgressAndPutOperationsForBackup(backupStore, operations, backup.Name); err != nil { removeIfComplete = false return err } @@ -346,7 +261,7 @@ func (c *asyncBackupOperationsReconciler) updateBackupAndOperationsJSON( removeIfComplete = false return errors.Wrapf(err, "error updating Backup %s", backup.Name) } - } else if asyncCompletionChanges { + } else if completionChanges { // If backup is still incomplete and no new errors are found but there are some new operations // completed, patch backup to reflect new completion numbers, but don't upload detailed json file err := c.Client.Patch(ctx, backup, client.MergeFrom(original)) @@ -357,41 +272,6 @@ func (c *asyncBackupOperationsReconciler) updateBackupAndOperationsJSON( return nil } -// returns a deep copy so we can minimize the time the map is locked -func (c *asyncBackupOperationsReconciler) getOperationsForBackup( - backupStore persistence.BackupStore, - backupName string) (*operationsForBackup, error) { - var err error - // lock operations map - c.itemOperationsMap.opsLock.Lock() - defer c.itemOperationsMap.opsLock.Unlock() - - operations, ok := c.itemOperationsMap.operations[backupName] - if !ok || len(operations.operations) == 0 { - operations = &operationsForBackup{} - operations.operations, err = backupStore.GetBackupItemOperations(backupName) - if err == nil { - c.itemOperationsMap.operations[backupName] = operations - } - } - return operations.DeepCopy(), err -} - -func (c *asyncBackupOperationsReconciler) putOperationsForBackup( - operations *operationsForBackup, - backupName string) { - if operations != nil { - c.itemOperationsMap.operations[backupName] = operations - } -} - -func (c *asyncBackupOperationsReconciler) deleteOperationsForBackup(backupName string) { - if _, ok := c.itemOperationsMap.operations[backupName]; ok { - delete(c.itemOperationsMap.operations, backupName) - } - return -} - func getBackupItemOperationProgress( backup *velerov1api.Backup, pluginManager clientmgmt.Manager, diff --git a/pkg/controller/async_backup_operations_controller_test.go b/pkg/controller/backup_operations_controller_test.go similarity index 81% rename from pkg/controller/async_backup_operations_controller_test.go rename to pkg/controller/backup_operations_controller_test.go index 935f92ac47..d7a7d98311 100644 --- a/pkg/controller/async_backup_operations_controller_test.go +++ b/pkg/controller/backup_operations_controller_test.go @@ -35,6 +35,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" @@ -51,20 +52,21 @@ var ( bia = &biav2mocks.BackupItemAction{} ) -func mockAsyncBackupOperationsReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock, freq time.Duration) (*asyncBackupOperationsReconciler, *BackupItemOperationsMap) { - abor, biaMap := NewAsyncBackupOperationsReconciler( +func mockBackupOperationsReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock, freq time.Duration) *backupOperationsReconciler { + abor := NewBackupOperationsReconciler( logrus.StandardLogger(), fakeClient, freq, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), + itemoperationmap.NewBackupItemOperationsMap(), ) abor.clock = fakeClock - return abor, biaMap + return abor } -func TestAsyncBackupOperationsReconcile(t *testing.T) { +func TestBackupOperationsReconcile(t *testing.T) { fakeClock := testclocks.NewFakeClock(time.Now()) metav1Now := metav1.NewTime(fakeClock.Now()) @@ -81,27 +83,27 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { expectPhase velerov1api.BackupPhase }{ { - name: "WaitingForPluginOperations backup with completed operations is FinalizingAfterPluginOperations", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + name: "WaitingForPluginOperations backup with completed operations is Finalizing", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). - ObjectMeta(builder.WithUID("foo")). + ObjectMeta(builder.WithUID("foo-11")). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), backupLocation: defaultBackupLocation, operationComplete: true, - expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + expectPhase: velerov1api.BackupPhaseFinalizing, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "backup-1", - BackupUID: "foo", - BackupItemAction: "foo", + BackupName: "backup-11", + BackupUID: "foo-11", + BackupItemAction: "foo-11", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, - OperationID: "operation-1", + OperationID: "operation-11", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, @@ -112,10 +114,10 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { }, { name: "WaitingForPluginOperations backup with incomplete operations is still incomplete", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-12"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). - ObjectMeta(builder.WithUID("foo")). + ObjectMeta(builder.WithUID("foo-12")). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), backupLocation: defaultBackupLocation, operationComplete: false, @@ -123,15 +125,15 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "backup-2", - BackupUID: "foo-2", - BackupItemAction: "foo-2", + BackupName: "backup-12", + BackupUID: "foo-12", + BackupItemAction: "foo-12", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, - OperationID: "operation-2", + OperationID: "operation-12", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, @@ -141,28 +143,28 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { }, }, { - name: "WaitingForPluginOperations backup with completed failed operations is FinalizingAfterPluginOperationsPartiallyFailed", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-3"). + name: "WaitingForPluginOperations backup with completed failed operations is FinalizingPartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-13"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). - ObjectMeta(builder.WithUID("foo")). + ObjectMeta(builder.WithUID("foo-13")). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), backupLocation: defaultBackupLocation, operationComplete: true, operationErr: "failed", - expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed, + expectPhase: velerov1api.BackupPhaseFinalizingPartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "backup-3", - BackupUID: "foo-3", - BackupItemAction: "foo-3", + BackupName: "backup-13", + BackupUID: "foo-13", + BackupItemAction: "foo-13", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, - OperationID: "operation-3", + OperationID: "operation-13", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, @@ -172,27 +174,27 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { }, }, { - name: "WaitingForPluginOperationsPartiallyFailed backup with completed operations is FinalizingAfterPluginOperationsPartiallyFailed", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + name: "WaitingForPluginOperationsPartiallyFailed backup with completed operations is FinalizingPartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-14"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). - ObjectMeta(builder.WithUID("foo")). + ObjectMeta(builder.WithUID("foo-14")). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), backupLocation: defaultBackupLocation, operationComplete: true, - expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed, + expectPhase: velerov1api.BackupPhaseFinalizingPartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "backup-4", - BackupUID: "foo-4", - BackupItemAction: "foo-4", + BackupName: "backup-14", + BackupUID: "foo-14", + BackupItemAction: "foo-14", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, - OperationID: "operation-4", + OperationID: "operation-14", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, @@ -203,10 +205,10 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { }, { name: "WaitingForPluginOperationsPartiallyFailed backup with incomplete operations is still incomplete", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-15"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). - ObjectMeta(builder.WithUID("foo")). + ObjectMeta(builder.WithUID("foo-15")). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), backupLocation: defaultBackupLocation, operationComplete: false, @@ -214,15 +216,15 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "backup-5", - BackupUID: "foo-5", - BackupItemAction: "foo-5", + BackupName: "backup-15", + BackupUID: "foo-15", + BackupItemAction: "foo-15", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, - OperationID: "operation-5", + OperationID: "operation-15", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, @@ -232,28 +234,28 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { }, }, { - name: "WaitingForPluginOperationsPartiallyFailed backup with completed failed operations is FinalizingAfterPluginOperationsPartiallyFailed", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-3"). + name: "WaitingForPluginOperationsPartiallyFailed backup with completed failed operations is FinalizingPartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-16"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). - ObjectMeta(builder.WithUID("foo")). + ObjectMeta(builder.WithUID("foo-16")). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), backupLocation: defaultBackupLocation, operationComplete: true, operationErr: "failed", - expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed, + expectPhase: velerov1api.BackupPhaseFinalizingPartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "backup-6", - BackupUID: "foo-6", - BackupItemAction: "foo-6", + BackupName: "backup-16", + BackupUID: "foo-16", + BackupItemAction: "foo-16", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, - OperationID: "operation-6", + OperationID: "operation-16", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, @@ -278,7 +280,7 @@ func TestAsyncBackupOperationsReconcile(t *testing.T) { } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) - reconciler, _ := mockAsyncBackupOperationsReconciler(fakeClient, fakeClock, defaultAsyncBackupOperationsFrequency) + reconciler := mockBackupOperationsReconciler(fakeClient, fakeClock, defaultBackupOperationsFrequency) pluginManager.On("CleanupClients").Return(nil) backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) backupStore.On("PutBackupItemOperations", mock.Anything, mock.Anything).Return(nil) diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index e8114abcf8..4edc03e554 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -149,23 +149,15 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || - backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed { + backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed || + backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { - log.Debugf("Skipping non-expired WaitingForPluginOperations backup %v", backup.Name) + log.Debugf("Skipping non-expired incomplete backup %v", backup.Name) continue } - log.Debug("WaitingForPluginOperations Backup is past expiration, syncing for garbage collection") - backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed - } - if backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed { - - if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { - log.Debugf("Skipping non-expired FinalizingAfterPluginOperations backup %v", backup.Name) - continue - } - log.Debug("FinalizingAfterPluginOperations Backup is past expiration, syncing for garbage collection") + log.Debugf("%v Backup is past expiration, syncing for garbage collection", backup.Status.Phase) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed } backup.Namespace = b.namespace diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index 44d40b729c..14f076cf4e 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -216,17 +216,17 @@ var _ = Describe("Backup Sync Reconciler", func() { }, { backup: builder.ForBackup("ns-1", "backup-4"). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + Phase(velerov1api.BackupPhaseFinalizing).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-5"). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed).Result(), + Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-6"). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + Phase(velerov1api.BackupPhaseFinalizing).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, @@ -262,19 +262,19 @@ var _ = Describe("Backup Sync Reconciler", func() { }, { backup: builder.ForBackup("ns-1", "backup-4"). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations). + Phase(velerov1api.BackupPhaseFinalizing). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-5"). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed). + Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-6"). - Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations). + Phase(velerov1api.BackupPhaseFinalizing). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), diff --git a/pkg/controller/constants.go b/pkg/controller/constants.go index f13edc0330..e38978f1c5 100644 --- a/pkg/controller/constants.go +++ b/pkg/controller/constants.go @@ -17,7 +17,7 @@ limitations under the License. package controller const ( - AsyncBackupOperations = "async-backup-operations" + BackupOperations = "backup-operations" Backup = "backup" BackupDeletion = "backup-deletion" BackupFinalizer = "backup-finalizer" @@ -35,7 +35,7 @@ const ( // DisableableControllers is a list of controllers that can be disabled var DisableableControllers = []string{ - AsyncBackupOperations, + BackupOperations, Backup, BackupDeletion, BackupFinalizer, diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 2eaa13f43f..11969a9eb8 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -28,6 +28,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" ) @@ -42,7 +43,7 @@ type downloadRequestReconciler struct { backupStoreGetter persistence.ObjectBackupStoreGetter // used to force update of async backup item operations before processing download request - backupItemOperationsMap *BackupItemOperationsMap + backupItemOperationsMap *itemoperationmap.BackupItemOperationsMap log logrus.FieldLogger } @@ -54,7 +55,7 @@ func NewDownloadRequestReconciler( newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, log logrus.FieldLogger, - backupItemOperationsMap *BackupItemOperationsMap, + backupItemOperationsMap *itemoperationmap.BackupItemOperationsMap, ) *downloadRequestReconciler { return &downloadRequestReconciler{ client: client, @@ -164,8 +165,8 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, errors.WithStack(err) } - // If this is a request for backup item operations, force update of in-memory operations that - // are not yet uploaded + // If this is a request for backup item operations, force upload of in-memory operations that + // are not yet uploaded (if there are any) if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindBackupItemOperations && r.backupItemOperationsMap != nil { // ignore errors here. If we can't upload anything here, process the download as usual diff --git a/pkg/itemoperation/backup_operation.go b/pkg/itemoperation/backup_operation.go index 5c226f13e4..520a255763 100644 --- a/pkg/itemoperation/backup_operation.go +++ b/pkg/itemoperation/backup_operation.go @@ -61,8 +61,8 @@ type BackupOperationSpec struct { // OperationID returned by the BIA plugin OperationID string "json:operationID" - // Items needing update after all async operations have completed - ItemsToUpdate []velero.ResourceIdentifier "json:itemsToUpdate" + // Items needing to be added to the backup after all async operations have completed + PostOperationItems []velero.ResourceIdentifier "json:postOperationItems" } func (in *BackupOperationSpec) DeepCopy() *BackupOperationSpec { @@ -77,8 +77,8 @@ func (in *BackupOperationSpec) DeepCopy() *BackupOperationSpec { func (in *BackupOperationSpec) DeepCopyInto(out *BackupOperationSpec) { *out = *in in.ResourceIdentifier.DeepCopyInto(&out.ResourceIdentifier) - if in.ItemsToUpdate != nil { - in, out := &in.ItemsToUpdate, &out.ItemsToUpdate + if in.PostOperationItems != nil { + in, out := &in.PostOperationItems, &out.PostOperationItems *out = make([]velero.ResourceIdentifier, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) diff --git a/pkg/itemoperationmap/backup_operation_map.go b/pkg/itemoperationmap/backup_operation_map.go new file mode 100644 index 0000000000..0fb369f33b --- /dev/null +++ b/pkg/itemoperationmap/backup_operation_map.go @@ -0,0 +1,170 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package itemoperationmap + +import ( + "bytes" + "sync" + + "github.com/pkg/errors" + + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/persistence" + "github.com/vmware-tanzu/velero/pkg/util/encode" +) + +type BackupItemOperationsMap struct { + opsMap map[string]*OperationsForBackup + opsLock sync.Mutex +} + +// Returns a pointer to a new BackupItemOperationsMap +func NewBackupItemOperationsMap() *BackupItemOperationsMap { + return &BackupItemOperationsMap{opsMap: make(map[string]*OperationsForBackup)} +} + +// returns a deep copy so we can minimize the time the map is locked +func (m *BackupItemOperationsMap) GetOperationsForBackup( + backupStore persistence.BackupStore, + backupName string) (*OperationsForBackup, error) { + var err error + // lock operations map + m.opsLock.Lock() + defer m.opsLock.Unlock() + + operations, ok := m.opsMap[backupName] + if !ok || len(operations.Operations) == 0 { + operations = &OperationsForBackup{} + operations.Operations, err = backupStore.GetBackupItemOperations(backupName) + if err == nil { + m.opsMap[backupName] = operations + } + } + return operations.DeepCopy(), err +} + +func (m *BackupItemOperationsMap) PutOperationsForBackup( + operations *OperationsForBackup, + backupName string) { + // lock operations map + m.opsLock.Lock() + defer m.opsLock.Unlock() + if operations != nil { + m.opsMap[backupName] = operations + } +} + +func (m *BackupItemOperationsMap) DeleteOperationsForBackup(backupName string) { + // lock operations map + m.opsLock.Lock() + defer m.opsLock.Unlock() + if _, ok := m.opsMap[backupName]; ok { + delete(m.opsMap, backupName) + } + return +} + +// UploadProgressAndPutOperationsForBackup will upload the item operations for this backup to +// the object store and update the map for this backup with the modified operations +func (m *BackupItemOperationsMap) UploadProgressAndPutOperationsForBackup( + backupStore persistence.BackupStore, + operations *OperationsForBackup, + backupName string) error { + + m.opsLock.Lock() + defer m.opsLock.Unlock() + + if operations == nil { + return errors.New("nil operations passed in") + } + if err := operations.uploadProgress(backupStore, backupName); err != nil { + return err + } + m.opsMap[backupName] = operations + return nil +} + +// UpdateForBackup will upload the item operations for this backup to +// the object store, if it has changes not yet uploaded +func (m *BackupItemOperationsMap) UpdateForBackup(backupStore persistence.BackupStore, backupName string) error { + // lock operations map + m.opsLock.Lock() + defer m.opsLock.Unlock() + + operations, ok := m.opsMap[backupName] + // if operations for this backup aren't found, or if there are no changes + // or errors since last update, do nothing + if !ok || (!operations.ChangesSinceUpdate && len(operations.ErrsSinceUpdate) == 0) { + return nil + } + if err := operations.uploadProgress(backupStore, backupName); err != nil { + return err + } + return nil +} + +type OperationsForBackup struct { + Operations []*itemoperation.BackupOperation + ChangesSinceUpdate bool + ErrsSinceUpdate []string +} + +func (in *OperationsForBackup) DeepCopy() *OperationsForBackup { + if in == nil { + return nil + } + out := new(OperationsForBackup) + in.DeepCopyInto(out) + return out +} + +func (in *OperationsForBackup) DeepCopyInto(out *OperationsForBackup) { + *out = *in + if in.Operations != nil { + in, out := &in.Operations, &out.Operations + *out = make([]*itemoperation.BackupOperation, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(itemoperation.BackupOperation) + (*in).DeepCopyInto(*out) + } + } + } + if in.ErrsSinceUpdate != nil { + in, out := &in.ErrsSinceUpdate, &out.ErrsSinceUpdate + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +func (o *OperationsForBackup) uploadProgress(backupStore persistence.BackupStore, backupName string) error { + if len(o.Operations) > 0 { + var backupItemOperations *bytes.Buffer + backupItemOperations, errs := encode.EncodeToJSONGzip(o.Operations, "backup item operations list") + if errs != nil { + return errors.Wrap(errs[0], "error encoding item operations json") + } + err := backupStore.PutBackupItemOperations(backupName, backupItemOperations) + if err != nil { + return errors.Wrap(err, "error uploading item operations json") + } + } + o.ChangesSinceUpdate = false + o.ErrsSinceUpdate = nil + return nil +} diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go index b02c35f3c4..d5a154b5fb 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go @@ -118,9 +118,9 @@ func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup * additionalItems = append(additionalItems, newItem) } - var itemsToUpdate []velero.ResourceIdentifier + var postOperationItems []velero.ResourceIdentifier - for _, itm := range res.ItemsToUpdate { + for _, itm := range res.PostOperationItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, @@ -130,10 +130,10 @@ func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup * Name: itm.Name, } - itemsToUpdate = append(itemsToUpdate, newItem) + postOperationItems = append(postOperationItems, newItem) } - return &updatedItem, additionalItems, res.OperationID, itemsToUpdate, nil + return &updatedItem, additionalItems, res.OperationID, postOperationItems, nil } func (c *BackupItemActionGRPCClient) Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) { diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go index 246214e0ac..6bb046cf58 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go @@ -107,7 +107,7 @@ func (s *BackupItemActionGRPCServer) Execute( return nil, common.NewGRPCError(errors.WithStack(err)) } - updatedItem, additionalItems, operationID, itemsToUpdate, err := impl.Execute(&item, &backup) + updatedItem, additionalItems, operationID, postOperationItems, err := impl.Execute(&item, &backup) if err != nil { return nil, common.NewGRPCError(err) } @@ -132,8 +132,8 @@ func (s *BackupItemActionGRPCServer) Execute( for _, item := range additionalItems { res.AdditionalItems = append(res.AdditionalItems, backupResourceIdentifierToProto(item)) } - for _, item := range itemsToUpdate { - res.ItemsToUpdate = append(res.ItemsToUpdate, backupResourceIdentifierToProto(item)) + for _, item := range postOperationItems { + res.PostOperationItems = append(res.PostOperationItems, backupResourceIdentifierToProto(item)) } return res, nil diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go index a0c91d4e16..82d70a8e84 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go @@ -91,16 +91,16 @@ func TestBackupItemActionGRPCServerExecute(t *testing.T) { require.NoError(t, err) tests := []struct { - name string - backup []byte - item []byte - implUpdatedItem runtime.Unstructured - implAdditionalItems []velero.ResourceIdentifier - implOperationID string - implItemsToUpdate []velero.ResourceIdentifier - implError error - expectError bool - skipMock bool + name string + backup []byte + item []byte + implUpdatedItem runtime.Unstructured + implAdditionalItems []velero.ResourceIdentifier + implOperationID string + implPostOperationItems []velero.ResourceIdentifier + implError error + expectError bool + skipMock bool }{ { name: "error unmarshaling item", @@ -154,7 +154,7 @@ func TestBackupItemActionGRPCServerExecute(t *testing.T) { defer itemAction.AssertExpectations(t) if !test.skipMock { - itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implOperationID, test.implItemsToUpdate, test.implError) + itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implOperationID, test.implPostOperationItems, test.implError) } s := &BackupItemActionGRPCServer{mux: &common.ServerMux{ diff --git a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go index 66c08e1c86..0bd8d3eb18 100644 --- a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go +++ b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go @@ -99,10 +99,10 @@ type ExecuteResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` - AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` - OperationID string `protobuf:"bytes,3,opt,name=operationID,proto3" json:"operationID,omitempty"` - ItemsToUpdate []*generated.ResourceIdentifier `protobuf:"bytes,4,rep,name=itemsToUpdate,proto3" json:"itemsToUpdate,omitempty"` + Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` + OperationID string `protobuf:"bytes,3,opt,name=operationID,proto3" json:"operationID,omitempty"` + PostOperationItems []*generated.ResourceIdentifier `protobuf:"bytes,4,rep,name=postOperationItems,proto3" json:"postOperationItems,omitempty"` } func (x *ExecuteResponse) Reset() { @@ -158,9 +158,9 @@ func (x *ExecuteResponse) GetOperationID() string { return "" } -func (x *ExecuteResponse) GetItemsToUpdate() []*generated.ResourceIdentifier { +func (x *ExecuteResponse) GetPostOperationItems() []*generated.ResourceIdentifier { if x != nil { - return x.ItemsToUpdate + return x.PostOperationItems } return nil } @@ -446,7 +446,7 @@ var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x22, 0xd5, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, @@ -455,67 +455,68 @@ var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x43, 0x0a, 0x0d, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x54, 0x6f, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0d, 0x69, 0x74, 0x65, 0x6d, - 0x73, 0x54, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, - 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, - 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, - 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, - 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x22, 0x73, 0x0a, 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, - 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, - 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, - 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, - 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, - 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, - 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, - 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, + 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x12, 0x70, 0x6f, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, + 0x12, 0x70, 0x6f, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, + 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, - 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, - 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, + 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, + 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x73, 0x0a, + 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, + 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, + 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, + 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, + 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, + 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, + 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, + 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, - 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x65, 0x12, 0x12, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, - 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, - 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, - 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, - 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, - 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x12, 0x2e, + 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, + 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, + 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, + 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, + 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, + 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -546,7 +547,7 @@ var file_backupitemaction_v2_BackupItemAction_proto_goTypes = []interface{}{ } var file_backupitemaction_v2_BackupItemAction_proto_depIdxs = []int32{ 7, // 0: v2.ExecuteResponse.additionalItems:type_name -> generated.ResourceIdentifier - 7, // 1: v2.ExecuteResponse.itemsToUpdate:type_name -> generated.ResourceIdentifier + 7, // 1: v2.ExecuteResponse.postOperationItems:type_name -> generated.ResourceIdentifier 8, // 2: v2.BackupItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector 9, // 3: v2.BackupItemActionProgressResponse.progress:type_name -> generated.OperationProgress 2, // 4: v2.BackupItemAction.AppliesTo:input_type -> v2.BackupItemActionAppliesToRequest diff --git a/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto b/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto index 366141dcad..36f067fe24 100644 --- a/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto +++ b/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto @@ -16,7 +16,7 @@ message ExecuteResponse { bytes item = 1; repeated generated.ResourceIdentifier additionalItems = 2; string operationID = 3; - repeated generated.ResourceIdentifier itemsToUpdate = 4; + repeated generated.ResourceIdentifier postOperationItems = 4; } service BackupItemAction { diff --git a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go index 920f07965e..32cd2af4ab 100644 --- a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go +++ b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go @@ -44,11 +44,17 @@ type BackupItemAction interface { // including mutating the item itself prior to backup. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying // additional related items that should be backed up now, an optional operationID for actions which - // initiate asynchronous actions, and a second slice of ResourceIdentifiers specifying related items - // which should be backed up after all asynchronous operations have completed. This last field will be + // initiate (asynchronous) operations, and a second slice of ResourceIdentifiers specifying related items + // which should be backed up after all operations have completed. This last field will be // ignored if operationID is empty, and should not be filled in unless the resource must be updated in the - // backup after async operations complete (i.e. some of the item's kubernetes metadata will be updated - // during the asynch operation which will be required during restore) + // backup after operations complete (i.e. some of the item's kubernetes metadata will be updated + // during the operation which will be required during restore) + // Note that (async) operations are not supported for items being backed up during Finalize phases, + // so a plugin should not return an OperationID if the backup phase is "Finalizing" + // or "FinalizingPartiallyFailed". The plugin should check the incoming + // backup.Status.Phase before initiating operations, since the backup has already passed the waiting + // for plugin operations phase. Plugins being called during Finalize will only be called for resources + // that were returned as postOperationItems. Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) // Progress allows the BackupItemAction to report on progress of an asynchronous action. diff --git a/pkg/util/encode/encode.go b/pkg/util/encode/encode.go index eee3932384..6e2b614016 100644 --- a/pkg/util/encode/encode.go +++ b/pkg/util/encode/encode.go @@ -18,6 +18,8 @@ package encode import ( "bytes" + "compress/gzip" + "encoding/json" "fmt" "io" @@ -70,3 +72,26 @@ func EncoderFor(format string, obj runtime.Object) (runtime.Encoder, error) { encoder = scheme.Codecs.EncoderForVersion(encoder, v1.SchemeGroupVersion) return encoder, nil } + +// EncodeToJSONGzip takes arbitrary Go data and encodes it to GZip compressed JSON in a buffer, as well as a description of the data to put into an error should encoding fail. +func EncodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, []error) { + buf := new(bytes.Buffer) + gzw := gzip.NewWriter(buf) + + // Since both encoding and closing the gzip writer could fail separately and both errors are useful, + // collect both errors to report back. + errs := []error{} + + if err := json.NewEncoder(gzw).Encode(data); err != nil { + errs = append(errs, errors.Wrapf(err, "error encoding %s", desc)) + } + if err := gzw.Close(); err != nil { + errs = append(errs, errors.Wrapf(err, "error closing gzip writer for %s", desc)) + } + + if len(errs) > 0 { + return nil, errs + } + + return buf, nil +} diff --git a/site/content/docs/main/api-types/backup.md b/site/content/docs/main/api-types/backup.md index ea9447b9ad..69c515c440 100644 --- a/site/content/docs/main/api-types/backup.md +++ b/site/content/docs/main/api-types/backup.md @@ -36,7 +36,7 @@ spec: # ItemOperationTimeout specifies the time used to wait for # asynchronous BackupItemAction operations # The default value is 1 hour. - csiSnapshotTimeout: 1h + itemOperationTimeout: 1h # Array of namespaces to include in the backup. If unspecified, all namespaces are included. # Optional. includedNamespaces: @@ -151,7 +151,7 @@ status: # The current phase. # Valid values are New, FailedValidation, InProgress, WaitingForPluginOperations, # WaitingForPluginOperationsPartiallyFailed, FinalizingafterPluginOperations, - # FinalizingAfterPluginOperationsPartiallyFailed, Completed, PartiallyFailed, Failed. + # FinalizingPartiallyFailed, Completed, PartiallyFailed, Failed. phase: "" # An array of any validation errors encountered. validationErrors: null @@ -163,12 +163,12 @@ status: volumeSnapshotsAttempted: 2 # Number of volume snapshots that Velero successfully created for this backup. volumeSnapshotsCompleted: 1 - # Number of attempted async BackupItemAction operations for this backup. - asyncBackupItemOperationsAttempted: 2 - # Number of async BackupItemAction operations that Velero successfully completed for this backup. - asyncBackupItemOperationsCompleted: 1 - # Number of async BackupItemAction operations that ended in failure for this backup. - asyncBackupItemOperationsFailed: 0 + # Number of attempted BackupItemAction operations for this backup. + backupItemOperationsAttempted: 2 + # Number of BackupItemAction operations that Velero successfully completed for this backup. + backupItemOperationsCompleted: 1 + # Number of BackupItemAction operations that ended in failure for this backup. + backupItemOperationsFailed: 0 # Number of warnings that were logged by the backup. warnings: 2 # Number of errors that were logged by the backup.