From ef8fa4f6be0757088160283a047031e4bfb6b758 Mon Sep 17 00:00:00 2001 From: Almog Gavra Date: Wed, 8 Jul 2020 08:58:41 -0700 Subject: [PATCH] fix: circumvent KAFKA-10179 by forcing changelog topics for tables (#5781) --- .../ksql/integration/WindowingIntTest.java | 2 +- .../ksql/planner/plan/DataSourceNodeTest.java | 2 + .../ksql/planner/plan/JoinNodeTest.java | 5 +- .../ksql/planner/plan/PlanTestUtil.java | 1 + .../ksql/execution/plan/TableSource.java | 16 +- .../ksql/execution/plan/TableSourceTest.java | 17 +- .../6.1.0_1594164253961/plan.json | 203 ++++++++++++ .../6.1.0_1594164253961/spec.json | 117 +++++++ .../6.1.0_1594164253961/topology | 45 +++ .../6.1.0_1594164260726/plan.json | 169 ++++++++++ .../6.1.0_1594164260726/spec.json | 143 ++++++++ .../6.1.0_1594164260726/topology | 43 +++ .../6.1.0_1594164260878/plan.json | 169 ++++++++++ .../6.1.0_1594164260878/spec.json | 145 ++++++++ .../6.1.0_1594164260878/topology | 43 +++ .../6.1.0_1594164259493/plan.json | 169 ++++++++++ .../6.1.0_1594164259493/spec.json | 155 +++++++++ .../6.1.0_1594164259493/topology | 43 +++ .../6.1.0_1594164259781/plan.json | 169 ++++++++++ .../6.1.0_1594164259781/spec.json | 143 ++++++++ .../6.1.0_1594164259781/topology | 43 +++ .../6.1.0_1594164259939/plan.json | 169 ++++++++++ .../6.1.0_1594164259939/spec.json | 145 ++++++++ .../6.1.0_1594164259939/topology | 43 +++ .../6.1.0_1594164258483/plan.json | 169 ++++++++++ .../6.1.0_1594164258483/spec.json | 155 +++++++++ .../6.1.0_1594164258483/topology | 43 +++ .../6.1.0_1594164258645/plan.json | 169 ++++++++++ .../6.1.0_1594164258645/spec.json | 143 ++++++++ .../6.1.0_1594164258645/topology | 43 +++ .../6.1.0_1594164258821/plan.json | 169 ++++++++++ .../6.1.0_1594164258821/spec.json | 145 ++++++++ .../6.1.0_1594164258821/topology | 43 +++ .../6.1.0_1594164258995/plan.json | 169 ++++++++++ .../6.1.0_1594164258995/spec.json | 155 +++++++++ .../6.1.0_1594164258995/topology | 43 +++ .../6.1.0_1594164259165/plan.json | 169 ++++++++++ .../6.1.0_1594164259165/spec.json | 143 ++++++++ .../6.1.0_1594164259165/topology | 43 +++ .../6.1.0_1594164259325/plan.json | 169 ++++++++++ .../6.1.0_1594164259325/spec.json | 145 ++++++++ .../6.1.0_1594164259325/topology | 43 +++ .../6.1.0_1594164260142/plan.json | 169 ++++++++++ .../6.1.0_1594164260142/spec.json | 161 +++++++++ .../6.1.0_1594164260142/topology | 43 +++ .../6.1.0_1594164260318/plan.json | 169 ++++++++++ .../6.1.0_1594164260318/spec.json | 149 +++++++++ .../6.1.0_1594164260318/topology | 43 +++ .../6.1.0_1594164260506/plan.json | 169 ++++++++++ .../6.1.0_1594164260506/spec.json | 151 +++++++++ .../6.1.0_1594164260506/topology | 43 +++ .../6.1.0_1594164262573/plan.json | 169 ++++++++++ .../6.1.0_1594164262573/spec.json | 119 +++++++ .../6.1.0_1594164262573/topology | 43 +++ .../6.1.0_1594164262709/plan.json | 169 ++++++++++ .../6.1.0_1594164262709/spec.json | 115 +++++++ .../6.1.0_1594164262709/topology | 43 +++ .../6.1.0_1594164262851/plan.json | 176 ++++++++++ .../6.1.0_1594164262851/spec.json | 115 +++++++ .../6.1.0_1594164262851/topology | 52 +++ .../6.1.0_1594164263710/plan.json | 146 ++++++++ .../6.1.0_1594164263710/spec.json | 87 +++++ .../6.1.0_1594164263710/topology | 22 ++ .../6.1.0_1594164263651/plan.json | 138 ++++++++ .../6.1.0_1594164263651/spec.json | 81 +++++ .../6.1.0_1594164263651/topology | 22 ++ .../6.1.0_1594164275124/plan.json | 169 ++++++++++ .../6.1.0_1594164275124/spec.json | 135 ++++++++ .../6.1.0_1594164275124/topology | 43 +++ .../6.1.0_1594164273927/plan.json | 169 ++++++++++ .../6.1.0_1594164273927/spec.json | 127 +++++++ .../6.1.0_1594164273927/topology | 43 +++ .../6.1.0_1594164272968/plan.json | 169 ++++++++++ .../6.1.0_1594164272968/spec.json | 127 +++++++ .../6.1.0_1594164272968/topology | 43 +++ .../6.1.0_1594164273171/plan.json | 169 ++++++++++ .../6.1.0_1594164273171/spec.json | 127 +++++++ .../6.1.0_1594164273171/topology | 43 +++ .../6.1.0_1594164271436/plan.json | 169 ++++++++++ .../6.1.0_1594164271436/spec.json | 132 ++++++++ .../6.1.0_1594164271436/topology | 43 +++ .../6.1.0_1594164270151/plan.json | 169 ++++++++++ .../6.1.0_1594164270151/spec.json | 169 ++++++++++ .../6.1.0_1594164270151/topology | 43 +++ .../6.1.0_1594164270502/plan.json | 169 ++++++++++ .../6.1.0_1594164270502/spec.json | 246 ++++++++++++++ .../6.1.0_1594164270502/topology | 43 +++ .../6.1.0_1594164270691/plan.json | 169 ++++++++++ .../6.1.0_1594164270691/spec.json | 230 +++++++++++++ .../6.1.0_1594164270691/topology | 43 +++ .../6.1.0_1594164270894/plan.json | 169 ++++++++++ .../6.1.0_1594164270894/spec.json | 232 +++++++++++++ .../6.1.0_1594164270894/topology | 43 +++ .../6.1.0_1594164270332/plan.json | 169 ++++++++++ .../6.1.0_1594164270332/spec.json | 169 ++++++++++ .../6.1.0_1594164270332/topology | 43 +++ .../6.1.0_1594164272386/plan.json | 169 ++++++++++ .../6.1.0_1594164272386/spec.json | 132 ++++++++ .../6.1.0_1594164272386/topology | 43 +++ .../6.1.0_1594164272529/plan.json | 169 ++++++++++ .../6.1.0_1594164272529/spec.json | 132 ++++++++ .../6.1.0_1594164272529/topology | 43 +++ .../6.1.0_1594164274741/plan.json | 169 ++++++++++ .../6.1.0_1594164274741/spec.json | 131 ++++++++ .../6.1.0_1594164274741/topology | 43 +++ .../6.1.0_1594164275959/plan.json | 235 +++++++++++++ .../6.1.0_1594164275959/spec.json | 219 ++++++++++++ .../6.1.0_1594164275959/topology | 78 +++++ .../6.1.0_1594164275418/plan.json | 169 ++++++++++ .../6.1.0_1594164275418/spec.json | 103 ++++++ .../6.1.0_1594164275418/topology | 43 +++ .../6.1.0_1594164275740/plan.json | 169 ++++++++++ .../6.1.0_1594164275740/spec.json | 62 ++++ .../6.1.0_1594164275740/topology | 43 +++ .../6.1.0_1594164268203/plan.json | 169 ++++++++++ .../6.1.0_1594164268203/spec.json | 233 +++++++++++++ .../6.1.0_1594164268203/topology | 43 +++ .../6.1.0_1594164268429/plan.json | 169 ++++++++++ .../6.1.0_1594164268429/spec.json | 233 +++++++++++++ .../6.1.0_1594164268429/topology | 43 +++ .../6.1.0_1594164273466/plan.json | 169 ++++++++++ .../6.1.0_1594164273466/spec.json | 147 +++++++++ .../6.1.0_1594164273466/topology | 43 +++ .../6.1.0_1594164271818/plan.json | 169 ++++++++++ .../6.1.0_1594164271818/spec.json | 132 ++++++++ .../6.1.0_1594164271818/topology | 43 +++ .../6.1.0_1594164272162/plan.json | 169 ++++++++++ .../6.1.0_1594164272162/spec.json | 123 +++++++ .../6.1.0_1594164272162/topology | 43 +++ .../6.1.0_1594164274216/plan.json | 176 ++++++++++ .../6.1.0_1594164274216/spec.json | 127 +++++++ .../6.1.0_1594164274216/topology | 52 +++ .../6.1.0_1594164276371/plan.json | 169 ++++++++++ .../6.1.0_1594164276371/spec.json | 119 +++++++ .../6.1.0_1594164276371/topology | 43 +++ .../6.1.0_1594164276881/plan.json | 169 ++++++++++ .../6.1.0_1594164276881/spec.json | 208 ++++++++++++ .../6.1.0_1594164276881/topology | 43 +++ .../6.1.0_1594164277051/plan.json | 169 ++++++++++ .../6.1.0_1594164277051/spec.json | 188 +++++++++++ .../6.1.0_1594164277051/topology | 43 +++ .../6.1.0_1594164277207/plan.json | 169 ++++++++++ .../6.1.0_1594164277207/spec.json | 190 +++++++++++ .../6.1.0_1594164277207/topology | 43 +++ .../6.1.0_1594164292476/plan.json | 214 ++++++++++++ .../6.1.0_1594164292476/spec.json | 186 +++++++++++ .../6.1.0_1594164292476/topology | 36 ++ .../6.1.0_1594164292589/plan.json | 214 ++++++++++++ .../6.1.0_1594164292589/spec.json | 150 +++++++++ .../6.1.0_1594164292589/topology | 36 ++ .../6.1.0_1594164292694/plan.json | 214 ++++++++++++ .../6.1.0_1594164292694/spec.json | 154 +++++++++ .../6.1.0_1594164292694/topology | 36 ++ .../6.1.0_1594164292808/plan.json | 201 +++++++++++ .../6.1.0_1594164292808/spec.json | 189 +++++++++++ .../6.1.0_1594164292808/topology | 51 +++ .../6.1.0_1594164292945/plan.json | 201 +++++++++++ .../6.1.0_1594164292945/spec.json | 157 +++++++++ .../6.1.0_1594164292945/topology | 51 +++ .../6.1.0_1594164293090/plan.json | 201 +++++++++++ .../6.1.0_1594164293090/spec.json | 161 +++++++++ .../6.1.0_1594164293090/topology | 51 +++ .../6.1.0_1594164293221/plan.json | 207 ++++++++++++ .../6.1.0_1594164293221/spec.json | 195 +++++++++++ .../6.1.0_1594164293221/topology | 51 +++ .../6.1.0_1594164293356/plan.json | 207 ++++++++++++ .../6.1.0_1594164293356/spec.json | 159 +++++++++ .../6.1.0_1594164293356/topology | 51 +++ .../6.1.0_1594164293507/plan.json | 207 ++++++++++++ .../6.1.0_1594164293507/spec.json | 163 +++++++++ .../6.1.0_1594164293507/topology | 51 +++ .../6.1.0_1594164289267/plan.json | 203 ++++++++++++ .../6.1.0_1594164289267/spec.json | 135 ++++++++ .../6.1.0_1594164289267/topology | 45 +++ .../6.1.0_1594164286600/plan.json | 196 +++++++++++ .../6.1.0_1594164286600/spec.json | 201 +++++++++++ .../6.1.0_1594164286600/topology | 33 ++ .../6.1.0_1594164286698/plan.json | 196 +++++++++++ .../6.1.0_1594164286698/spec.json | 169 ++++++++++ .../6.1.0_1594164286698/topology | 33 ++ .../6.1.0_1594164286793/plan.json | 196 +++++++++++ .../6.1.0_1594164286793/spec.json | 173 ++++++++++ .../6.1.0_1594164286793/topology | 33 ++ .../6.1.0_1594164286326/plan.json | 196 +++++++++++ .../6.1.0_1594164286326/spec.json | 211 ++++++++++++ .../6.1.0_1594164286326/topology | 33 ++ .../6.1.0_1594164286416/plan.json | 196 +++++++++++ .../6.1.0_1594164286416/spec.json | 179 ++++++++++ .../6.1.0_1594164286416/topology | 33 ++ .../6.1.0_1594164286504/plan.json | 196 +++++++++++ .../6.1.0_1594164286504/spec.json | 183 +++++++++++ .../6.1.0_1594164286504/topology | 33 ++ .../6.1.0_1594164287813/plan.json | 200 +++++++++++ .../6.1.0_1594164287813/spec.json | 111 +++++++ .../6.1.0_1594164287813/topology | 33 ++ .../6.1.0_1594164287892/plan.json | 202 ++++++++++++ .../6.1.0_1594164287892/spec.json | 108 ++++++ .../6.1.0_1594164287892/topology | 33 ++ .../6.1.0_1594164287711/plan.json | 196 +++++++++++ .../6.1.0_1594164287711/spec.json | 149 +++++++++ .../6.1.0_1594164287711/topology | 33 ++ .../6.1.0_1594164291624/plan.json | 203 ++++++++++++ .../6.1.0_1594164291624/spec.json | 121 +++++++ .../6.1.0_1594164291624/topology | 45 +++ .../6.1.0_1594164286911/plan.json | 311 ++++++++++++++++++ .../6.1.0_1594164286911/spec.json | 169 ++++++++++ .../6.1.0_1594164286911/topology | 48 +++ .../6.1.0_1594164285187/plan.json | 189 +++++++++++ .../6.1.0_1594164285187/spec.json | 228 +++++++++++++ .../6.1.0_1594164285187/topology | 48 +++ .../6.1.0_1594164285316/plan.json | 189 +++++++++++ .../6.1.0_1594164285316/spec.json | 196 +++++++++++ .../6.1.0_1594164285316/topology | 48 +++ .../6.1.0_1594164285447/plan.json | 189 +++++++++++ .../6.1.0_1594164285447/spec.json | 200 +++++++++++ .../6.1.0_1594164285447/topology | 48 +++ .../6.1.0_1594164287151/plan.json | 196 +++++++++++ .../6.1.0_1594164287151/spec.json | 195 +++++++++++ .../6.1.0_1594164287151/topology | 57 ++++ .../6.1.0_1594164284630/plan.json | 189 +++++++++++ .../6.1.0_1594164284630/spec.json | 250 ++++++++++++++ .../6.1.0_1594164284630/topology | 48 +++ .../6.1.0_1594164284817/plan.json | 189 +++++++++++ .../6.1.0_1594164284817/spec.json | 218 ++++++++++++ .../6.1.0_1594164284817/topology | 48 +++ .../6.1.0_1594164285011/plan.json | 189 +++++++++++ .../6.1.0_1594164285011/spec.json | 222 +++++++++++++ .../6.1.0_1594164285011/topology | 48 +++ .../6.1.0_1594164285739/plan.json | 189 +++++++++++ .../6.1.0_1594164285739/spec.json | 250 ++++++++++++++ .../6.1.0_1594164285739/topology | 48 +++ .../6.1.0_1594164285943/plan.json | 189 +++++++++++ .../6.1.0_1594164285943/spec.json | 218 ++++++++++++ .../6.1.0_1594164285943/topology | 48 +++ .../6.1.0_1594164286127/plan.json | 189 +++++++++++ .../6.1.0_1594164286127/spec.json | 236 +++++++++++++ .../6.1.0_1594164286127/topology | 48 +++ .../6.1.0_1594164288094/plan.json | 193 +++++++++++ .../6.1.0_1594164288094/spec.json | 125 +++++++ .../6.1.0_1594164288094/topology | 48 +++ .../6.1.0_1594164288199/plan.json | 195 +++++++++++ .../6.1.0_1594164288199/spec.json | 122 +++++++ .../6.1.0_1594164288199/topology | 48 +++ .../6.1.0_1594164287985/plan.json | 189 +++++++++++ .../6.1.0_1594164287985/spec.json | 169 ++++++++++ .../6.1.0_1594164287985/topology | 48 +++ .../6.1.0_1594164279482/plan.json | 189 +++++++++++ .../6.1.0_1594164279482/spec.json | 128 +++++++ .../6.1.0_1594164279482/topology | 48 +++ .../6.1.0_1594164288627/plan.json | 189 +++++++++++ .../6.1.0_1594164288627/spec.json | 128 +++++++ .../6.1.0_1594164288627/topology | 48 +++ .../6.1.0_1594164294074/plan.json | 130 ++++++++ .../6.1.0_1594164294074/spec.json | 108 ++++++ .../6.1.0_1594164294074/topology | 22 ++ .../6.1.0_1594164294154/plan.json | 130 ++++++++ .../6.1.0_1594164294154/spec.json | 108 ++++++ .../6.1.0_1594164294154/topology | 22 ++ .../6.1.0_1594164293990/plan.json | 130 ++++++++ .../6.1.0_1594164293990/spec.json | 108 ++++++ .../6.1.0_1594164293990/topology | 22 ++ .../6.1.0_1594164293900/plan.json | 130 ++++++++ .../6.1.0_1594164293900/spec.json | 108 ++++++ .../6.1.0_1594164293900/topology | 22 ++ .../6.1.0_1594164293814/plan.json | 130 ++++++++ .../6.1.0_1594164293814/spec.json | 108 ++++++ .../6.1.0_1594164293814/topology | 22 ++ .../6.1.0_1594164296549/plan.json | 255 ++++++++++++++ .../6.1.0_1594164296549/spec.json | 140 ++++++++ .../6.1.0_1594164296549/topology | 50 +++ .../6.1.0_1594164296667/plan.json | 255 ++++++++++++++ .../6.1.0_1594164296667/spec.json | 140 ++++++++ .../6.1.0_1594164296667/topology | 50 +++ .../6.1.0_1594164300902/plan.json | 255 ++++++++++++++ .../6.1.0_1594164300902/spec.json | 148 +++++++++ .../6.1.0_1594164300902/topology | 50 +++ .../6.1.0_1594164299923/plan.json | 271 +++++++++++++++ .../6.1.0_1594164299923/spec.json | 279 ++++++++++++++++ .../6.1.0_1594164299923/topology | 68 ++++ .../6.1.0_1594164300054/plan.json | 271 +++++++++++++++ .../6.1.0_1594164300054/spec.json | 287 ++++++++++++++++ .../6.1.0_1594164300054/topology | 68 ++++ .../6.1.0_1594164299126/plan.json | 264 +++++++++++++++ .../6.1.0_1594164299126/spec.json | 191 +++++++++++ .../6.1.0_1594164299126/topology | 56 ++++ .../6.1.0_1594164299646/plan.json | 278 ++++++++++++++++ .../6.1.0_1594164299646/spec.json | 215 ++++++++++++ .../6.1.0_1594164299646/topology | 80 +++++ .../6.1.0_1594164299254/plan.json | 271 +++++++++++++++ .../6.1.0_1594164299254/spec.json | 202 ++++++++++++ .../6.1.0_1594164299254/topology | 68 ++++ .../6.1.0_1594164299387/plan.json | 278 ++++++++++++++++ .../6.1.0_1594164299387/spec.json | 275 ++++++++++++++++ .../6.1.0_1594164299387/topology | 80 +++++ .../6.1.0_1594164299805/plan.json | 264 +++++++++++++++ .../6.1.0_1594164299805/spec.json | 200 +++++++++++ .../6.1.0_1594164299805/topology | 56 ++++ .../6.1.0_1594164300181/plan.json | 264 +++++++++++++++ .../6.1.0_1594164300181/spec.json | 182 ++++++++++ .../6.1.0_1594164300181/topology | 56 ++++ .../6.1.0_1594164300302/plan.json | 264 +++++++++++++++ .../6.1.0_1594164300302/spec.json | 182 ++++++++++ .../6.1.0_1594164300302/topology | 56 ++++ .../6.1.0_1594164298243/plan.json | 255 ++++++++++++++ .../6.1.0_1594164298243/spec.json | 146 ++++++++ .../6.1.0_1594164298243/topology | 50 +++ .../6.1.0_1594164297093/plan.json | 255 ++++++++++++++ .../6.1.0_1594164297093/spec.json | 148 +++++++++ .../6.1.0_1594164297093/topology | 50 +++ .../6.1.0_1594164299012/plan.json | 255 ++++++++++++++ .../6.1.0_1594164299012/spec.json | 148 +++++++++ .../6.1.0_1594164299012/topology | 50 +++ .../6.1.0_1594164298814/plan.json | 255 ++++++++++++++ .../6.1.0_1594164298814/spec.json | 148 +++++++++ .../6.1.0_1594164298814/topology | 50 +++ .../6.1.0_1594164297706/plan.json | 262 +++++++++++++++ .../6.1.0_1594164297706/spec.json | 166 ++++++++++ .../6.1.0_1594164297706/topology | 62 ++++ .../6.1.0_1594164298916/plan.json | 262 +++++++++++++++ .../6.1.0_1594164298916/spec.json | 166 ++++++++++ .../6.1.0_1594164298916/topology | 62 ++++ .../6.1.0_1594164297985/plan.json | 269 +++++++++++++++ .../6.1.0_1594164297985/spec.json | 161 +++++++++ .../6.1.0_1594164297985/topology | 74 +++++ .../6.1.0_1594164297838/plan.json | 269 +++++++++++++++ .../6.1.0_1594164297838/spec.json | 161 +++++++++ .../6.1.0_1594164297838/topology | 74 +++++ .../6.1.0_1594164298399/plan.json | 255 ++++++++++++++ .../6.1.0_1594164298399/spec.json | 143 ++++++++ .../6.1.0_1594164298399/topology | 50 +++ .../6.1.0_1594164298620/plan.json | 255 ++++++++++++++ .../6.1.0_1594164298620/spec.json | 141 ++++++++ .../6.1.0_1594164298620/topology | 50 +++ .../6.1.0_1594164298520/plan.json | 255 ++++++++++++++ .../6.1.0_1594164298520/spec.json | 139 ++++++++ .../6.1.0_1594164298520/topology | 50 +++ .../6.1.0_1594164298715/plan.json | 255 ++++++++++++++ .../6.1.0_1594164298715/spec.json | 143 ++++++++ .../6.1.0_1594164298715/topology | 50 +++ .../6.1.0_1594164297464/plan.json | 255 ++++++++++++++ .../6.1.0_1594164297464/spec.json | 164 +++++++++ .../6.1.0_1594164297464/topology | 50 +++ .../6.1.0_1594164297333/plan.json | 255 ++++++++++++++ .../6.1.0_1594164297333/spec.json | 155 +++++++++ .../6.1.0_1594164297333/topology | 50 +++ .../6.1.0_1594164297215/plan.json | 255 ++++++++++++++ .../6.1.0_1594164297215/spec.json | 173 ++++++++++ .../6.1.0_1594164297215/topology | 50 +++ .../6.1.0_1594164300694/plan.json | 240 ++++++++++++++ .../6.1.0_1594164300694/spec.json | 174 ++++++++++ .../6.1.0_1594164300694/topology | 71 ++++ .../6.1.0_1594164300564/plan.json | 240 ++++++++++++++ .../6.1.0_1594164300564/spec.json | 165 ++++++++++ .../6.1.0_1594164300564/topology | 71 ++++ .../6.1.0_1594164296795/plan.json | 255 ++++++++++++++ .../6.1.0_1594164296795/spec.json | 140 ++++++++ .../6.1.0_1594164296795/topology | 50 +++ .../6.1.0_1594164303934/plan.json | 130 ++++++++ .../6.1.0_1594164303934/spec.json | 73 ++++ .../6.1.0_1594164303934/topology | 22 ++ .../6.1.0_1594164304009/plan.json | 203 ++++++++++++ .../6.1.0_1594164304009/spec.json | 116 +++++++ .../6.1.0_1594164304009/topology | 45 +++ .../6.1.0_1594164307333/plan.json | 228 +++++++++++++ .../6.1.0_1594164307333/spec.json | 227 +++++++++++++ .../6.1.0_1594164307333/topology | 69 ++++ .../6.1.0_1594164308603/plan.json | 130 ++++++++ .../6.1.0_1594164308603/spec.json | 81 +++++ .../6.1.0_1594164308603/topology | 22 ++ .../6.1.0_1594164308646/plan.json | 130 ++++++++ .../6.1.0_1594164308646/spec.json | 85 +++++ .../6.1.0_1594164308646/topology | 22 ++ .../6.1.0_1594164308696/plan.json | 130 ++++++++ .../6.1.0_1594164308696/spec.json | 81 +++++ .../6.1.0_1594164308696/topology | 22 ++ .../6.1.0_1594164308560/plan.json | 130 ++++++++ .../6.1.0_1594164308560/spec.json | 78 +++++ .../6.1.0_1594164308560/topology | 22 ++ .../6.1.0_1594164308509/plan.json | 130 ++++++++ .../6.1.0_1594164308509/spec.json | 88 +++++ .../6.1.0_1594164308509/topology | 22 ++ .../6.1.0_1594164309255/plan.json | 169 ++++++++++ .../6.1.0_1594164309255/spec.json | 135 ++++++++ .../6.1.0_1594164309255/topology | 43 +++ .../6.1.0_1594164309836/plan.json | 136 ++++++++ .../6.1.0_1594164309836/spec.json | 98 ++++++ .../6.1.0_1594164309836/topology | 25 ++ .../query-validation-tests/joins.json | 1 + .../query-validation-tests/table.json | 10 +- .../resources/ksql-plan-schema/schema.json | 3 + .../streams/ExecutionStepFactory.java | 3 +- .../ksql/execution/streams/SourceBuilder.java | 21 +- .../execution/streams/SourceBuilderTest.java | 41 ++- .../streams/StepSchemaResolverTest.java | 3 +- 394 files changed, 50856 insertions(+), 32 deletions(-) create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.1.0_1594164307333/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.1.0_1594164307333/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.1.0_1594164307333/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_NOT_reuse_source_topic_for_change_log_if_topology_optimizations_are_off/6.1.0_1594164308603/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_NOT_reuse_source_topic_for_change_log_if_topology_optimizations_are_off/6.1.0_1594164308603/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_NOT_reuse_source_topic_for_change_log_if_topology_optimizations_are_off/6.1.0_1594164308603/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_forward_nulls_in_changelog_when_table_not_materialized/6.1.0_1594164308646/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_forward_nulls_in_changelog_when_table_not_materialized/6.1.0_1594164308646/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_forward_nulls_in_changelog_when_table_not_materialized/6.1.0_1594164308646/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_not_blow_up_on_null_key/6.1.0_1594164308696/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_not_blow_up_on_null_key/6.1.0_1594164308696/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_not_blow_up_on_null_key/6.1.0_1594164308696/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_not_reuse_source_topic_for_change_log_by_default/6.1.0_1594164308560/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_not_reuse_source_topic_for_change_log_by_default/6.1.0_1594164308560/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_should_not_reuse_source_topic_for_change_log_by_default/6.1.0_1594164308560/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_update-delete/6.1.0_1594164308509/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_update-delete/6.1.0_1594164308509/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/table_-_update-delete/6.1.0_1594164308509/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.1.0_1594164309255/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.1.0_1594164309255/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.1.0_1594164309255/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.1.0_1594164309836/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.1.0_1594164309836/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.1.0_1594164309836/topology diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/integration/WindowingIntTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/integration/WindowingIntTest.java index 10bca064c7b1..4d7d92c7372d 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/integration/WindowingIntTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/integration/WindowingIntTest.java @@ -138,7 +138,7 @@ public void shouldAggregateWithNoWindow() { // Then: assertOutputOf(resultStream0, expected, is(expected)); assertTableCanBeUsedAsSource(expected, is(expected)); - assertTopicsCleanedUp(TopicCleanupPolicy.COMPACT, 4, resultStream0, resultStream1); + assertTopicsCleanedUp(TopicCleanupPolicy.COMPACT, 5, resultStream0, resultStream1); } @Test diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/DataSourceNodeTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/DataSourceNodeTest.java index d75bbfb966ed..43f6f03ba539 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/DataSourceNodeTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/DataSourceNodeTest.java @@ -68,6 +68,7 @@ import java.util.stream.Stream; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.TopologyDescription; @@ -171,6 +172,7 @@ public void before() { when(ksqlStreamBuilder.buildValueSerde(any(), any(), any())).thenReturn(rowSerde); when(ksqlStreamBuilder.getFunctionRegistry()).thenReturn(functionRegistry); + when(rowSerde.serializer()).thenReturn(mock(Serializer.class)); when(rowSerde.deserializer()).thenReturn(mock(Deserializer.class)); when(dataSource.getKsqlTopic()).thenReturn(topic); diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/JoinNodeTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/JoinNodeTest.java index 71f3178d5b80..76e0db1691de 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/JoinNodeTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/JoinNodeTest.java @@ -20,6 +20,7 @@ import static io.confluent.ksql.planner.plan.JoinNode.JoinType.LEFT; import static io.confluent.ksql.planner.plan.JoinNode.JoinType.OUTER; import static io.confluent.ksql.planner.plan.PlanTestUtil.SOURCE_NODE; +import static io.confluent.ksql.planner.plan.PlanTestUtil.SOURCE_NODE_FORCE_CHANGELOG; import static io.confluent.ksql.planner.plan.PlanTestUtil.getNodeByName; import static java.util.Optional.empty; import static org.hamcrest.CoreMatchers.equalTo; @@ -216,11 +217,11 @@ public void shouldBuildSourceNode() { setupTopicClientExpectations(1, 1); buildJoin(); final TopologyDescription.Source node = (TopologyDescription.Source) getNodeByName( - builder.build(), SOURCE_NODE); + builder.build(), SOURCE_NODE_FORCE_CHANGELOG); final List successors = node.successors().stream().map(TopologyDescription.Node::name) .collect(Collectors.toList()); assertThat(node.predecessors(), equalTo(Collections.emptySet())); - assertThat(successors, equalTo(Collections.singletonList("KTABLE-SOURCE-0000000001"))); + assertThat(successors, equalTo(Collections.singletonList("KTABLE-SOURCE-0000000002"))); assertThat(node.topicSet(), equalTo(ImmutableSet.of("test2"))); } diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/PlanTestUtil.java b/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/PlanTestUtil.java index 677787975fda..21421fe010a9 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/PlanTestUtil.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/planner/plan/PlanTestUtil.java @@ -31,6 +31,7 @@ final class PlanTestUtil { static final String TRANSFORM_NODE = "KSTREAM-TRANSFORMVALUES-0000000001"; static final String SOURCE_NODE = "KSTREAM-SOURCE-0000000000"; + static final String SOURCE_NODE_FORCE_CHANGELOG = "KSTREAM-SOURCE-0000000001"; private PlanTestUtil() { } diff --git a/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/TableSource.java b/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/TableSource.java index 096ebe599219..854ee2bdb8e6 100644 --- a/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/TableSource.java +++ b/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/TableSource.java @@ -26,15 +26,23 @@ @Immutable public final class TableSource extends SourceStep> { + private final Boolean forceChangelog; + public TableSource( @JsonProperty(value = "properties", required = true) final ExecutionStepPropertiesV1 properties, @JsonProperty(value = "topicName", required = true) final String topicName, @JsonProperty(value = "formats", required = true) final Formats formats, @JsonProperty("timestampColumn") final Optional timestampColumn, - @JsonProperty(value = "sourceSchema", required = true) final LogicalSchema sourceSchema + @JsonProperty(value = "sourceSchema", required = true) final LogicalSchema sourceSchema, + @JsonProperty(value = "forceChangelog") final Optional forceChangelog ) { super(properties, topicName, formats, timestampColumn, sourceSchema); + this.forceChangelog = forceChangelog.orElse(false); + } + + public Boolean isForceChangelog() { + return forceChangelog; } @Override @@ -55,11 +63,13 @@ public boolean equals(final Object o) { && Objects.equals(topicName, that.topicName) && Objects.equals(formats, that.formats) && Objects.equals(timestampColumn, that.timestampColumn) - && Objects.equals(sourceSchema, that.sourceSchema); + && Objects.equals(sourceSchema, that.sourceSchema) + && Objects.equals(forceChangelog, that.forceChangelog); } @Override public int hashCode() { - return Objects.hash(properties, topicName, formats, timestampColumn, sourceSchema); + return Objects.hash( + properties, topicName, formats, timestampColumn, sourceSchema, forceChangelog); } } diff --git a/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/TableSourceTest.java b/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/TableSourceTest.java index 64551add1d59..0baee6074efb 100644 --- a/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/TableSourceTest.java +++ b/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/TableSourceTest.java @@ -48,24 +48,27 @@ public void shouldImplementEquals() { new EqualsTester() .addEqualityGroup( new TableSource( - properties1, "topic1", formats1, Optional.of(timestamp1), schema1), + properties1, "topic1", formats1, Optional.of(timestamp1), schema1, Optional.of(false)), new TableSource( - properties1, "topic1", formats1, Optional.of(timestamp1), schema1)) + properties1, "topic1", formats1, Optional.of(timestamp1), schema1, Optional.of(false))) .addEqualityGroup( new TableSource( - properties2, "topic1", formats1, Optional.of(timestamp1), schema1)) + properties2, "topic1", formats1, Optional.of(timestamp1), schema1, Optional.of(false))) .addEqualityGroup( new TableSource( - properties1, "topic2", formats1, Optional.of(timestamp1), schema1)) + properties1, "topic2", formats1, Optional.of(timestamp1), schema1, Optional.of(false))) .addEqualityGroup( new TableSource( - properties1, "topic1", formats2, Optional.of(timestamp1), schema1)) + properties1, "topic1", formats2, Optional.of(timestamp1), schema1, Optional.of(false))) .addEqualityGroup( new TableSource( - properties1, "topic1", formats1, Optional.of(timestamp2), schema1)) + properties1, "topic1", formats1, Optional.of(timestamp2), schema1, Optional.of(false))) .addEqualityGroup( new TableSource( - properties1, "topic1", formats1, Optional.of(timestamp1), schema2)) + properties1, "topic1", formats1, Optional.of(timestamp1), schema2, Optional.of(false))) + .addEqualityGroup( + new TableSource( + properties1, "topic1", formats1, Optional.of(timestamp1), schema1, Optional.of(true))) .testEquals(); } } \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/plan.json new file mode 100644 index 000000000000..e2fdf39c500c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM I1 (ID INTEGER KEY, V0 INTEGER, V1 INTEGER) WITH (KAFKA_TOPIC='i1', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "I1", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "topicName" : "i1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE I2 (ID INTEGER PRIMARY KEY, V0 INTEGER, V1 INTEGER) WITH (KAFKA_TOPIC='i2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "I2", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "topicName" : "i2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM I1 I1\nINNER JOIN I2 I2 ON ((AS_VALUE(I1.ID) = I2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`I2_ID` INTEGER KEY, `I1_ID` INTEGER, `I1_V0` INTEGER, `I1_V1` INTEGER, `I2_V0` INTEGER, `I2_V1` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "I1", "I2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "i1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER" + }, + "keyExpression" : "AS_VALUE(ID)" + }, + "keyColumnNames" : [ "I1_KSQL_COL_0" ], + "selectExpressions" : [ "V0 AS I1_V0", "V1 AS I1_V1", "ROWTIME AS I1_ROWTIME", "ID AS I1_ID", "KSQL_COL_0 AS I1_KSQL_COL_0" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "i2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "I2_ID" ], + "selectExpressions" : [ "V0 AS I2_V0", "V1 AS I2_V1", "ROWTIME AS I2_ROWTIME", "ID AS I2_ID" ] + }, + "keyColName" : "I2_ID" + }, + "keyColumnNames" : [ "I2_ID" ], + "selectExpressions" : [ "I1_ID AS I1_ID", "I1_V0 AS I1_V0", "I1_V1 AS I1_V1", "I2_V0 AS I2_V0", "I2_V1 AS I2_V1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/spec.json new file mode 100644 index 000000000000..f2639d85bf77 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/spec.json @@ -0,0 +1,117 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164253961, + "path" : "query-validation-tests/as_value.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "join", + "inputs" : [ { + "topic" : "i2", + "key" : 1, + "value" : { + "V0" : 2, + "V1" : 3 + } + }, { + "topic" : "i1", + "key" : 1, + "value" : { + "V0" : 2, + "V1" : 3 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "I1_ID" : 1, + "I1_V0" : 2, + "I1_V1" : 3, + "I2_V0" : 2, + "I2_V1" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "i1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "i2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM I1 (ID INT KEY, V0 INT, V1 INT) WITH (kafka_topic='i1', value_format='JSON');", "CREATE TABLE I2 (ID INT PRIMARY KEY, V0 INT, V1 INT) WITH (kafka_topic='i2', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT * FROM I1 JOIN I2 ON AS_VALUE(I1.ID) = I2.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "I2_ID INT KEY, I1_ID INT, I1_V0 INT, I1_V1 INT, I2_V0 INT, I2_V1 INT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "i1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "i2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/topology new file mode 100644 index 000000000000..764f728ffa57 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_join/6.1.0_1594164253961/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [i2]) + --> KTABLE-SOURCE-0000000002 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Join + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [i1]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000006 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasLeft (stores: []) + --> Join-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- PrependAliasLeft + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/plan.json new file mode 100644 index 000000000000..e39064f4f62c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/spec.json new file mode 100644 index 000000000000..c96a23af041c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/spec.json @@ -0,0 +1,143 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164260726, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT> NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT, KSQL_INTERNAL_COL_2 BOOLEAN> NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT, KSQL_AGG_VARIABLE_0 ARRAY> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list bool map table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : false + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : false, + "key2" : true + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : true + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ false ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.1.0_1594164260726/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/plan.json new file mode 100644 index 000000000000..f1917c95ff4b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/spec.json new file mode 100644 index 000000000000..7a7dda78d9a2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/spec.json @@ -0,0 +1,145 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164260878, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT> NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT, KSQL_INTERNAL_COL_2 BOOLEAN> NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT, KSQL_AGG_VARIABLE_0 ARRAY> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list bool map table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : false + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : false, + "key2" : true + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : true + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ false ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n repeated ConnectDefault2Entry VALUE = 2;\n\n message ConnectDefault2Entry {\n string key = 1;\n bool value = 2;\n }\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.1.0_1594164260878/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/plan.json new file mode 100644 index 000000000000..ead036250fcd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/spec.json new file mode 100644 index 000000000000..6d349860b5a3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/spec.json @@ -0,0 +1,155 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164259493, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list double table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 5.4 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100.1 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500.9 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 300.8 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 5.4 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100.1 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500.9 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 300.8 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "double" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.1.0_1594164259493/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/plan.json new file mode 100644 index 000000000000..3a1274f021bd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/spec.json new file mode 100644 index 000000000000..a2b8664da61a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/spec.json @@ -0,0 +1,143 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164259781, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list double table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 5.4 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100.1 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500.9 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 300.8 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 5.4 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100.1 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500.9 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 300.8 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.1.0_1594164259781/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/plan.json new file mode 100644 index 000000000000..ddb01b27fb17 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/spec.json new file mode 100644 index 000000000000..f45fa2fdd5f2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/spec.json @@ -0,0 +1,145 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164259939, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list double table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 5.4 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100.1 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500.9 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 300.8 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 5.4 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100.1 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500.9 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 300.8 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n double VALUE = 1;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.1.0_1594164259939/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/plan.json new file mode 100644 index 000000000000..733c8b7b670d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/spec.json new file mode 100644 index 000000000000..0923156c8737 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/spec.json @@ -0,0 +1,155 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164258483, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list int table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.1.0_1594164258483/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/plan.json new file mode 100644 index 000000000000..e2b865309022 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/spec.json new file mode 100644 index 000000000000..0cda0dc37d0d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/spec.json @@ -0,0 +1,143 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164258645, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list int table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.1.0_1594164258645/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/plan.json new file mode 100644 index 000000000000..db9722cce89c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/spec.json new file mode 100644 index 000000000000..9ebcf0369173 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/spec.json @@ -0,0 +1,145 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164258821, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list int table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 VALUE = 1;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.1.0_1594164258821/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/plan.json new file mode 100644 index 000000000000..5a07e45bf8eb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/spec.json new file mode 100644 index 000000000000..93ee2b2bbb2c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/spec.json @@ -0,0 +1,155 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164258995, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list long table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.1.0_1594164258995/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/plan.json new file mode 100644 index 000000000000..e93f6f20c5f8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/spec.json new file mode 100644 index 000000000000..6c68d28927df --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/spec.json @@ -0,0 +1,143 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164259165, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list long table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.1.0_1594164259165/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/plan.json new file mode 100644 index 000000000000..19459b135a7d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/spec.json new file mode 100644 index 000000000000..d12cd0fb95a7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/spec.json @@ -0,0 +1,145 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164259325, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list long table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 VALUE = 1;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.1.0_1594164259325/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/plan.json new file mode 100644 index 000000000000..adb32117e910 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/spec.json new file mode 100644 index 000000000000..8688dff1ebe6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/spec.json @@ -0,0 +1,161 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164260142, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list string table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "foo" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "bar" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "foo" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.1.0_1594164260142/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/plan.json new file mode 100644 index 000000000000..84d7ae381844 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/spec.json new file mode 100644 index 000000000000..a6bf8b9e8fe2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164260318, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list string table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "foo" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "bar" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "foo" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.1.0_1594164260318/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/plan.json new file mode 100644 index 000000000000..a64b86c73740 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/spec.json new file mode 100644 index 000000000000..70ac30b687d7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/spec.json @@ -0,0 +1,151 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164260506, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_S2_0.S2" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "collect_list string table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "foo" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "bar" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "foo" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string VALUE = 1;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/topology new file mode 100644 index 000000000000..ad94f5da9404 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.1.0_1594164260506/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/plan.json new file mode 100644 index 000000000000..dc738156868d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (ID STRING PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='input_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`ID` STRING KEY, `NAME` STRING", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.NAME NAME,\n COUNT(1) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INPUT.NAME\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NAME` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` STRING KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "NAME AS NAME", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "NAME" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "NAME" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/spec.json new file mode 100644 index 000000000000..7cde457ac4c8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/spec.json @@ -0,0 +1,119 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164262573, + "path" : "query-validation-tests/count.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "count table", + "inputs" : [ { + "topic" : "input_topic", + "key" : "0", + "value" : "bob" + }, { + "topic" : "input_topic", + "key" : "0", + "value" : "john" + }, { + "topic" : "input_topic", + "key" : "100", + "value" : "john" + }, { + "topic" : "input_topic", + "key" : "100", + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "bob", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "bob", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "john", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "john", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "john", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (ID STRING PRIMARY KEY, name STRING) WITH (kafka_topic='input_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT as SELECT NAME, count(1) FROM input group by name;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "input_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/topology new file mode 100644 index 000000000000..fc3495845270 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.1.0_1594164262573/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [input_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/plan.json new file mode 100644 index 000000000000..55bd2dff97fb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.ID ID,\n COUNT(*) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INPUT.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/spec.json new file mode 100644 index 000000000000..e8df5f06553e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/spec.json @@ -0,0 +1,115 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164262709, + "path" : "query-validation-tests/count.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "should count back to zero", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "0" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID bigint) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT as SELECT ID, COUNT() FROM INPUT GROUP BY ID;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.1.0_1594164262709/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/plan.json new file mode 100644 index 000000000000..3f0778dd1bd5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/plan.json @@ -0,0 +1,176 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.ID ID,\n COUNT(*) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INPUT.ID\nHAVING (COUNT(*) > 0)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)", "COUNT(ROWTIME)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 0)" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/spec.json new file mode 100644 index 000000000000..adbaf6cbb453 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/spec.json @@ -0,0 +1,115 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164262851, + "path" : "query-validation-tests/count.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "should support removing zero counts from table", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : null + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID bigint) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT as SELECT ID, COUNT() FROM INPUT GROUP BY ID HAVING COUNT() > 0;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/topology new file mode 100644 index 000000000000..3883a49c9ff6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.1.0_1594164262851/topology @@ -0,0 +1,52 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/plan.json new file mode 100644 index 000000000000..d7c34780ce5f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/plan.json @@ -0,0 +1,146 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TYPE ADDRESS AS STRUCT;", + "ddlCommand" : { + "@type" : "registerTypeV1", + "type" : "STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>", + "typeName" : "ADDRESS" + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TYPE PERSON AS STRUCT>;", + "ddlCommand" : { + "@type" : "registerTypeV1", + "type" : "STRUCT<`NAME` STRING, `ADDRESS` STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>>", + "typeName" : "PERSON" + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE PEOPLE (K STRING PRIMARY KEY, PERSON STRUCT>) WITH (KAFKA_TOPIC='test', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "PEOPLE", + "schema" : "`K` STRING KEY, `PERSON` STRUCT<`NAME` STRING, `ADDRESS` STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>>", + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COPY AS SELECT *\nFROM PEOPLE PEOPLE\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COPY", + "schema" : "`K` STRING KEY, `PERSON` STRUCT<`NAME` STRING, `ADDRESS` STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>>", + "topicName" : "COPY", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "PEOPLE" ], + "sink" : "COPY", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COPY" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `PERSON` STRUCT<`NAME` STRING, `ADDRESS` STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>>", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "PERSON AS PERSON" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "COPY" + }, + "queryId" : "CTAS_COPY_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/spec.json new file mode 100644 index 000000000000..df6ac14f4294 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/spec.json @@ -0,0 +1,87 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164263710, + "path" : "query-validation-tests/create-type.json", + "schemas" : { + "CTAS_COPY_0.KsqlTopic.Source" : "STRUCT>> NOT NULL", + "CTAS_COPY_0.COPY" : "STRUCT>> NOT NULL" + }, + "testCase" : { + "name" : "create nested type", + "inputs" : [ { + "topic" : "test", + "key" : "a", + "value" : { + "person" : { + "name" : "jay", + "address" : { + "number" : 899, + "street" : "W. Evelyn", + "city" : "Mountain View" + } + } + } + } ], + "outputs" : [ { + "topic" : "COPY", + "key" : "a", + "value" : { + "PERSON" : { + "NAME" : "jay", + "ADDRESS" : { + "NUMBER" : 899, + "STREET" : "W. Evelyn", + "CITY" : "Mountain View" + } + } + } + } ], + "topics" : [ { + "name" : "test", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COPY", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TYPE ADDRESS AS STRUCT;", "CREATE TYPE PERSON AS STRUCT;", "CREATE TABLE people (K STRING PRIMARY KEY, person PERSON) WITH (kafka_topic='test', value_format='JSON');", "CREATE TABLE copy AS SELECT * FROM people;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "COPY", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COPY_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/topology new file mode 100644 index 000000000000..a3ab73c4049b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_nested_type/6.1.0_1594164263710/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: COPY) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/plan.json new file mode 100644 index 000000000000..9fb01c8a23f6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/plan.json @@ -0,0 +1,138 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TYPE ADDRESS AS STRUCT;", + "ddlCommand" : { + "@type" : "registerTypeV1", + "type" : "STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>", + "typeName" : "ADDRESS" + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE ADDRESSES (K STRING PRIMARY KEY, ADDRESS STRUCT) WITH (KAFKA_TOPIC='test', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "ADDRESSES", + "schema" : "`K` STRING KEY, `ADDRESS` STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>", + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COPY AS SELECT *\nFROM ADDRESSES ADDRESSES\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COPY", + "schema" : "`K` STRING KEY, `ADDRESS` STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>", + "topicName" : "COPY", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "ADDRESSES" ], + "sink" : "COPY", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COPY" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `ADDRESS` STRUCT<`NUMBER` INTEGER, `STREET` STRING, `CITY` STRING>", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ADDRESS AS ADDRESS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "COPY" + }, + "queryId" : "CTAS_COPY_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/spec.json new file mode 100644 index 000000000000..104c4924f0e5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/spec.json @@ -0,0 +1,81 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164263651, + "path" : "query-validation-tests/create-type.json", + "schemas" : { + "CTAS_COPY_0.KsqlTopic.Source" : "STRUCT
> NOT NULL", + "CTAS_COPY_0.COPY" : "STRUCT
> NOT NULL" + }, + "testCase" : { + "name" : "create simple type", + "inputs" : [ { + "topic" : "test", + "key" : "a", + "value" : { + "address" : { + "number" : 899, + "street" : "W. Evelyn", + "city" : "Mountain View" + } + } + } ], + "outputs" : [ { + "topic" : "COPY", + "key" : "a", + "value" : { + "ADDRESS" : { + "NUMBER" : 899, + "STREET" : "W. Evelyn", + "CITY" : "Mountain View" + } + } + } ], + "topics" : [ { + "name" : "test", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COPY", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TYPE ADDRESS AS STRUCT;", "CREATE TABLE addresses (K STRING PRIMARY KEY, address ADDRESS) WITH (kafka_topic='test', value_format='JSON');", "CREATE TABLE copy AS SELECT * FROM addresses;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "COPY", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COPY_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/topology new file mode 100644 index 000000000000..a3ab73c4049b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/create-type_-_create_simple_type/6.1.0_1594164263651/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: COPY) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/plan.json new file mode 100644 index 000000000000..37e9dd2d27a7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, D0 INTEGER, D1 STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `D0` INTEGER, `D1` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.D1 D1,\n SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INTEGER), 1) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.D1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`D1` STRING KEY, `KSQL_COL_0` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `D0` INTEGER, `D1` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "D1 AS D1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "D1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "D1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "D1" ], + "selectExpressions" : [ "SUBSTRING('Mr Bugalicious', CAST(KSQL_AGG_VARIABLE_0 AS INTEGER), 1) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/spec.json new file mode 100644 index 000000000000..22b2d04cb3f8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/spec.json @@ -0,0 +1,135 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164275124, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "UDAF nested in UDF in select expression (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : "0,x" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,x" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,xxx" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,xxx" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,yy" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "x", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "x", + "value" : "r" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "r" + }, { + "topic" : "OUTPUT", + "key" : "x", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "yy", + "value" : "M" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, d0 INT, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT d1, SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INT), 1) FROM TEST GROUP BY d1;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.1.0_1594164275124/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/plan.json new file mode 100644 index 000000000000..b581171b45f7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, F0 INTEGER, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (TEST.F0 - TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY (TEST.F0 - TEST.F1)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F0 AS F0", "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "(F0 - F1)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F0", "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/spec.json new file mode 100644 index 000000000000..07dfdc720ab5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/spec.json @@ -0,0 +1,127 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164273927, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "arithmetic binary expression with projection in-order & non-commutative group by (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,0" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,1" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,1" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "4,2" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f0 - f1, COUNT(*) FROM TEST GROUP BY f0 - f1;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.1.0_1594164273927/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/plan.json new file mode 100644 index 000000000000..d777ac2f643a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/spec.json new file mode 100644 index 000000000000..f39bbb2e5cc8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/spec.json @@ -0,0 +1,127 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164272968, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "constant (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "3" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT 1, COUNT(*) FROM TEST GROUP BY 1;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.1.0_1594164272968/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/plan.json new file mode 100644 index 000000000000..31d6faaf4327 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.REGION REGION,\n SUBSTRING(TEST.REGION, 2, 1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` STRING, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "SUBSTRING(REGION, 2, 1) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/spec.json new file mode 100644 index 000000000000..01e7e5e97395 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/spec.json @@ -0,0 +1,127 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164273171, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "field with field used in function in projection (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1,1" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1,0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region, SUBSTRING(region, 2, 1), COUNT(*) FROM TEST GROUP BY region;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.1.0_1594164273171/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/plan.json new file mode 100644 index 000000000000..e6af04d54b00 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.REGION REGION,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/spec.json new file mode 100644 index 000000000000..01c56debe78e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/spec.json @@ -0,0 +1,132 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164271436, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "field with re-key (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region, COUNT(*) FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "REGION STRING KEY, KSQL_COL_0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.1.0_1594164271436/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/plan.json new file mode 100644 index 000000000000..733f05f6ad9b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_1` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/spec.json new file mode 100644 index 000000000000..982e1fa1d4e5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164270151, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "fields (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "1,b" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "1,a" + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : "1,a,0,1" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : "2,b,0,1" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : "1,a,0,0" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : "1,b,0,1" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : "2,b,0,0" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : "1,b,0,0" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : "1,a,0,1" + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.1.0_1594164270151/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/plan.json new file mode 100644 index 000000000000..d61758a21243 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_1` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/spec.json new file mode 100644 index 000000000000..c3fa3bb95720 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/spec.json @@ -0,0 +1,246 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164270502, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "fields (table->table) - format - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : { + "F1" : 2, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : { + "F1" : 1, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : { + "F1" : 2, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : { + "F1" : 1, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.1.0_1594164270502/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/plan.json new file mode 100644 index 000000000000..bffe13aeb3f2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_1` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/spec.json new file mode 100644 index 000000000000..93205eedeb2e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/spec.json @@ -0,0 +1,230 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164270691, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "fields (table->table) - format - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : { + "F1" : 2, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : { + "F1" : 1, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : { + "F1" : 2, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : { + "F1" : 1, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.1.0_1594164270691/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/plan.json new file mode 100644 index 000000000000..69943582e400 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_1` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/spec.json new file mode 100644 index 000000000000..9ffd2707f706 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/spec.json @@ -0,0 +1,232 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164270894, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "fields (table->table) - format - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : { + "F1" : 2, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : { + "F1" : 1, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : { + "F1" : 2, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : { + "F1" : 1, + "F2" : "b", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 0 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : { + "F1" : 1, + "F2" : "a", + "ROWTIME" : 0, + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.1.0_1594164270894/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/plan.json new file mode 100644 index 000000000000..09ed8e153455 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n AS_VALUE(TEST.F1) F3,\n AS_VALUE(TEST.F2) F4,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_1` STRING KEY, `F3` INTEGER, `F4` STRING, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_1" ], + "selectExpressions" : [ "AS_VALUE(F1) AS F3", "AS_VALUE(F2) AS F4", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/spec.json new file mode 100644 index 000000000000..adf5b025fc30 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164270332, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "fields - copied into value (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "1,b" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "1,a" + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : "1,a,0,1" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : "2,b,0,1" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : "1,a,0,0" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : "1,b,0,1" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|2", + "value" : "2,b,0,0" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "b|+|1", + "value" : "1,b,0,0" + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "a|+|1", + "value" : "1,a,0,1" + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : "1,a,1" + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : "2,b,1" + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : "1,a,0" + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : "1,b,1" + }, { + "topic" : "OUTPUT", + "key" : "b|+|2", + "value" : "2,b,0" + }, { + "topic" : "OUTPUT", + "key" : "b|+|1", + "value" : "1,b,0" + }, { + "topic" : "OUTPUT", + "key" : "a|+|1", + "value" : "1,a,1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, AS_VALUE(f1) AS F3, AS_VALUE(F2) AS F4, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "KSQL_COL_1 STRING KEY, F3 INT, F4 STRING, KSQL_COL_0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.1.0_1594164270332/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/plan.json new file mode 100644 index 000000000000..93ded1ceb1d3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUBSTRING(TEST.REGION, 7, 2) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY SUBSTRING(TEST.REGION, 7, 2)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "SUBSTRING(REGION, 7, 2)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/spec.json new file mode 100644 index 000000000000..ee3c3971e3ed --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/spec.json @@ -0,0 +1,132 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164272386, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "function (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,prefixr0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,prefixr1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,prefixr0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,prefixr0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT SUBSTRING(region, 7, 2), COUNT(*) FROM TEST GROUP BY SUBSTRING(region, 7, 2);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "KSQL_COL_0 STRING KEY, KSQL_COL_1 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.1.0_1594164272386/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/plan.json new file mode 100644 index 000000000000..fce3e2d55c9f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEN(TEST.REGION) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY LEN(TEST.REGION)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "LEN(REGION)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/spec.json new file mode 100644 index 000000000000..701600f90d4f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/spec.json @@ -0,0 +1,132 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164272529, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "int function (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "usa" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "eu" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "usa" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "usa" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT LEN(region), COUNT(*) FROM TEST GROUP BY LEN(region);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "KSQL_COL_0 INT KEY, KSQL_COL_1 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.1.0_1594164272529/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/plan.json new file mode 100644 index 000000000000..6e7dc73f3201 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/spec.json new file mode 100644 index 000000000000..40af9b40d0fd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/spec.json @@ -0,0 +1,131 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164274741, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "missing matching projection field (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,b" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "a", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "a", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT F2, COUNT(*) FROM TEST GROUP BY f2;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.1.0_1594164274741/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/plan.json new file mode 100644 index 000000000000..cd0bff106894 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/plan.json @@ -0,0 +1,235 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, TOTAL INTEGER) WITH (KAFKA_TOPIC='T1', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, TOTAL INTEGER) WITH (KAFKA_TOPIC='T2', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n SUM((T1.TOTAL + (CASE WHEN (T2.TOTAL IS NULL) THEN 0 ELSE T2.TOTAL END))) SUM\nFROM T1 T1\nLEFT OUTER JOIN T2 T2 ON ((T1.ID = T2.ID))\nGROUP BY T1.ID\nHAVING (COUNT(1) > 0)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `SUM` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "TOTAL AS T1_TOTAL", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "TOTAL AS T2_TOTAL", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_ID AS T1_ID", "T1_TOTAL AS T1_TOTAL", "T2_TOTAL AS T2_TOTAL", "(T1_TOTAL + (CASE WHEN (T2_TOTAL IS NULL) THEN 0 ELSE T2_TOTAL END)) AS KSQL_INTERNAL_COL_3", "1 AS KSQL_INTERNAL_COL_4" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "T1_ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "T1_ID", "T1_TOTAL", "T2_TOTAL" ], + "aggregationFunctions" : [ "SUM(KSQL_INTERNAL_COL_3)", "COUNT(KSQL_INTERNAL_COL_4)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 0)" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS SUM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/spec.json new file mode 100644 index 000000000000..9301ae456bb9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/spec.json @@ -0,0 +1,219 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164275959, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "on join", + "inputs" : [ { + "topic" : "T1", + "key" : 0, + "value" : { + "total" : 100 + } + }, { + "topic" : "T1", + "key" : 1, + "value" : { + "total" : 101 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "total" : 5 + } + }, { + "topic" : "T2", + "key" : 1, + "value" : { + "total" : 10 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "total" : 20 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 100 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "SUM" : 101 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 105 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "SUM" : 111 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 120 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 100 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T1", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T2", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE t1 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T1', value_format='AVRO');", "CREATE TABLE t2 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T2', value_format='AVRO');", "CREATE TABLE OUTPUT AS SELECT t1.ID, SUM(t1.total + CASE WHEN t2.total IS NULL THEN 0 ELSE t2.total END) as SUM FROM T1 LEFT JOIN T2 ON (t1.ID = t2.ID) GROUP BY t1.ID HAVING COUNT(1) > 0;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "T1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "T2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/topology new file mode 100644 index 000000000000..932d1ce7e1f9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.1.0_1594164275959/topology @@ -0,0 +1,78 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [T1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [T2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [T1-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: [T2-STATE-STORE-0000000006]) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-FILTER-0000000016 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000018 + <-- KTABLE-FILTER-0000000016 + Sink: KSTREAM-SINK-0000000018 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000019 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000020 + Processor: KTABLE-AGGREGATE-0000000020 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000019 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KTABLE-AGGREGATE-0000000020 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000026 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000026 (stores: []) + --> KSTREAM-SINK-0000000027 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000027 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000026 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/plan.json new file mode 100644 index 000000000000..659a6e5b0508 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, STR STRING, POS INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUBSTRING(TEST.STR, TEST.POS) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY SUBSTRING(TEST.STR, TEST.POS)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "STR AS STR", "POS AS POS", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "SUBSTRING(STR, POS)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "STR", "POS", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/spec.json new file mode 100644 index 000000000000..22432b4ee9d7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/spec.json @@ -0,0 +1,103 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164275418, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "should exclude any table row whose single GROUP BY expression resolves to NULL", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "xx,1" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "x," + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "xx,1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "xx", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "xx", + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, str STRING, pos INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT SUBSTRING(str, pos), COUNT() FROM TEST GROUP BY SUBSTRING(str, pos);" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.1.0_1594164275418/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/plan.json new file mode 100644 index 000000000000..56dea7bc0705 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n BAD_UDF(TEST.ID) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY BAD_UDF(TEST.ID)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "BAD_UDF(ID)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/spec.json new file mode 100644 index 000000000000..38180bc2c847 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/spec.json @@ -0,0 +1,62 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164275740, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "should exclude any table row whose single GROUP BY expression throws", + "inputs" : [ { + "topic" : "test_topic", + "key" : "2", + "value" : "1" + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, id INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT BAD_UDF(id), COUNT() FROM TEST GROUP BY BAD_UDF(id);" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.1.0_1594164275740/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/plan.json new file mode 100644 index 000000000000..33dd0aa62eba --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA NEW_KEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/spec.json new file mode 100644 index 000000000000..84ad7b571744 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/spec.json @@ -0,0 +1,233 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164268203, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "single column with alias (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "d1" + } + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : "d1", + "value" : { + "ROWTIME" : 0, + "DATA" : "d1" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : "d2", + "value" : { + "ROWTIME" : 0, + "DATA" : "d2" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : "d1", + "value" : { + "ROWTIME" : 0, + "DATA" : "d1" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : "d2", + "value" : { + "ROWTIME" : 0, + "DATA" : "d2" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : "d1", + "value" : { + "ROWTIME" : 0, + "DATA" : "d1" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "d1", + "value" : { + "ROWTIME" : 0, + "DATA" : "d1", + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "d2", + "value" : { + "ROWTIME" : 0, + "DATA" : "d2", + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "d1", + "value" : { + "ROWTIME" : 0, + "DATA" : "d1", + "KSQL_AGG_VARIABLE_0" : 2 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "d2", + "value" : { + "ROWTIME" : 0, + "DATA" : "d2", + "KSQL_AGG_VARIABLE_0" : 2 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : "d1", + "value" : { + "ROWTIME" : 0, + "DATA" : "d1", + "KSQL_AGG_VARIABLE_0" : 3 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "NEW_KEY STRING KEY, COUNT BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.1.0_1594164268203/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/plan.json new file mode 100644 index 000000000000..7af02beb43c8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEN(TEST.DATA) NEW_KEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY LEN(TEST.DATA)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "LEN(DATA)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/spec.json new file mode 100644 index 000000000000..a8426c9c0a26 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/spec.json @@ -0,0 +1,233 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164268429, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "single expression with alias (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "22" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "333" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "-2" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "003" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "2-" + } + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : 2, + "value" : { + "ROWTIME" : 0, + "DATA" : "22" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : 3, + "value" : { + "ROWTIME" : 0, + "DATA" : "333" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : 2, + "value" : { + "ROWTIME" : 0, + "DATA" : "-2" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : 3, + "value" : { + "ROWTIME" : 0, + "DATA" : "003" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "key" : 2, + "value" : { + "ROWTIME" : 0, + "DATA" : "2-" + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : 2, + "value" : { + "ROWTIME" : 0, + "DATA" : "22", + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : 3, + "value" : { + "ROWTIME" : 0, + "DATA" : "333", + "KSQL_AGG_VARIABLE_0" : 1 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : 2, + "value" : { + "ROWTIME" : 0, + "DATA" : "-2", + "KSQL_AGG_VARIABLE_0" : 2 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : 3, + "value" : { + "ROWTIME" : 0, + "DATA" : "003", + "KSQL_AGG_VARIABLE_0" : 2 + } + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "key" : 2, + "value" : { + "ROWTIME" : 0, + "DATA" : "2-", + "KSQL_AGG_VARIABLE_0" : 3 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT LEN(DATA) AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY LEN(DATA);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "NEW_KEY INT KEY, COUNT BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.1.0_1594164268429/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/plan.json new file mode 100644 index 000000000000..f0b8287e7735 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, USER INTEGER, SUBREGION STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `USER` INTEGER, `SUBREGION` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (TEST.REGION + TEST.SUBREGION) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY (TEST.REGION + TEST.SUBREGION)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `USER` INTEGER, `SUBREGION` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "SUBREGION AS SUBREGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "(REGION + SUBREGION)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "SUBREGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/spec.json new file mode 100644 index 000000000000..1fc027740e39 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/spec.json @@ -0,0 +1,147 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164273466, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "string concat using + op (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a,r0" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,a,r1" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,a,r0" + }, { + "topic" : "test_topic", + "key" : "4", + "value" : "4,b,r0" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,a,r0" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b,r1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0b", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1a", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1b", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, user INT, subregion VARCHAR, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region + subregion, COUNT(*) FROM TEST GROUP BY region + subregion;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.1.0_1594164273466/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/plan.json new file mode 100644 index 000000000000..0ecbeeffb00c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.REGION REGION,\n (COUNT(*) * 2) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "(KSQL_AGG_VARIABLE_0 * 2) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/spec.json new file mode 100644 index 000000000000..9eb5e3ef2861 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/spec.json @@ -0,0 +1,132 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164271818, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "with aggregate arithmetic (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "4" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "4" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region, COUNT(*) * 2 FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "REGION STRING KEY, KSQL_COL_0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.1.0_1594164271818/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/plan.json new file mode 100644 index 000000000000..c22e274985a2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F0 INTEGER, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F0` INTEGER, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F0 F0,\n (TEST.F0 * SUM(TEST.F1)) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F0\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F0` INTEGER KEY, `KSQL_COL_0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F0` INTEGER, `F1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F0 AS F0", "F1 AS F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F0", "F1" ], + "aggregationFunctions" : [ "SUM(F1)" ] + }, + "keyColumnNames" : [ "F0" ], + "selectExpressions" : [ "(F0 * KSQL_AGG_VARIABLE_0) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/spec.json new file mode 100644 index 000000000000..ea35e0f2bd90 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/spec.json @@ -0,0 +1,123 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164272162, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "with aggregate arithmetic involving source field (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 2, + "value" : "2,10" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,20" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,30" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : "20" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "40" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "60" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f0, f0 * SUM(f1) FROM TEST GROUP BY f0;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.1.0_1594164272162/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/plan.json new file mode 100644 index 000000000000..e879bd1fa68c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/plan.json @@ -0,0 +1,176 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, F0 INTEGER, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n SUM(TEST.F0) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F1\nHAVING (COUNT(TEST.F1) > 0)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F1 AS F1", "F0 AS F0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F0" ], + "aggregationFunctions" : [ "SUM(F0)", "COUNT(F1)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 0)" + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/spec.json new file mode 100644 index 000000000000..3a4e57a6e72f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/spec.json @@ -0,0 +1,127 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164274216, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "with having expression (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,0" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,1" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,0" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : "4" + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : "3" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : "5" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, SUM(f0) FROM TEST GROUP BY f1 HAVING COUNT(f1) > 0;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/topology new file mode 100644 index 000000000000..3883a49c9ff6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.1.0_1594164274216/topology @@ -0,0 +1,52 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/plan.json new file mode 100644 index 000000000000..9f8acfc6d57f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (ID INTEGER PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 K,\n COUNT(1) ID\nFROM INPUT INPUT\nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "1 AS KSQL_INTERNAL_COL_0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_0)" ] + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/spec.json new file mode 100644 index 000000000000..0184a32b0cfa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/spec.json @@ -0,0 +1,119 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164276371, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "zero non-agg columns (table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 10, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 1666, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 98, + "value" : { + "VALUE" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (ID INT PRIMARY KEY, VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT 1 as k, count(1) AS ID FROM INPUT group by 1;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/topology new file mode 100644 index 000000000000..399d12f468bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.1.0_1594164276371/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/plan.json new file mode 100644 index 000000000000..7b263d21b0a9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID BIGINT, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COUNT_BY_REGION AS SELECT\n TEST.REGION REGION,\n HISTOGRAM(TEST.NAME) COUNTS\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COUNT_BY_REGION", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "topicName" : "COUNT_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "COUNT_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COUNT_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "REGION", "NAME" ], + "aggregationFunctions" : [ "HISTOGRAM(NAME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "COUNT_BY_REGION" + }, + "queryId" : "CTAS_COUNT_BY_REGION_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/spec.json new file mode 100644 index 000000000000..7a65aa599b75 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/spec.json @@ -0,0 +1,208 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164276881, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_COUNT_BY_REGION_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_COUNT_BY_REGION_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_COUNT_BY_REGION_0.COUNT_BY_REGION" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "histogram on a table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "alice", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "ID" : 2, + "NAME" : "carol", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "ID" : 3, + "NAME" : "dave", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + } ], + "outputs" : [ { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "REGION", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "COUNT_BY_REGION", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/topology new file mode 100644 index 000000000000..e0b9acf25713 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.1.0_1594164276881/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: COUNT_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/plan.json new file mode 100644 index 000000000000..4b610681c505 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID BIGINT, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COUNT_BY_REGION AS SELECT\n TEST.REGION REGION,\n HISTOGRAM(TEST.NAME) COUNTS\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COUNT_BY_REGION", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "topicName" : "COUNT_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "COUNT_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COUNT_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "REGION", "NAME" ], + "aggregationFunctions" : [ "HISTOGRAM(NAME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "COUNT_BY_REGION" + }, + "queryId" : "CTAS_COUNT_BY_REGION_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/spec.json new file mode 100644 index 000000000000..375dc9fc0f3b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/spec.json @@ -0,0 +1,188 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164277051, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_COUNT_BY_REGION_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_COUNT_BY_REGION_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_COUNT_BY_REGION_0.COUNT_BY_REGION" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "histogram on a table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "alice", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "ID" : 2, + "NAME" : "carol", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "ID" : 3, + "NAME" : "dave", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + } ], + "outputs" : [ { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "COUNT_BY_REGION", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/topology new file mode 100644 index 000000000000..e0b9acf25713 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.1.0_1594164277051/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: COUNT_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/plan.json new file mode 100644 index 000000000000..0d78f8b35faf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/plan.json @@ -0,0 +1,169 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID BIGINT, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COUNT_BY_REGION AS SELECT\n TEST.REGION REGION,\n HISTOGRAM(TEST.NAME) COUNTS\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COUNT_BY_REGION", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "topicName" : "COUNT_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "COUNT_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COUNT_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "REGION", "NAME" ], + "aggregationFunctions" : [ "HISTOGRAM(NAME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "COUNT_BY_REGION" + }, + "queryId" : "CTAS_COUNT_BY_REGION_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/spec.json new file mode 100644 index 000000000000..148e0b15a86e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/spec.json @@ -0,0 +1,190 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164277207, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_COUNT_BY_REGION_0.Aggregate.GroupBy" : "STRUCT NOT NULL", + "CTAS_COUNT_BY_REGION_0.Aggregate.Aggregate.Materialize" : "STRUCT> NOT NULL", + "CTAS_COUNT_BY_REGION_0.COUNT_BY_REGION" : "STRUCT> NOT NULL" + }, + "testCase" : { + "name" : "histogram on a table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "alice", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "ID" : 2, + "NAME" : "carol", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "ID" : 3, + "NAME" : "dave", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + } ], + "outputs" : [ { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n string NAME = 2;\n string REGION = 3;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "COUNT_BY_REGION", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/topology new file mode 100644 index 000000000000..e0b9acf25713 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.1.0_1594164277207/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: [test_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: COUNT_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/plan.json new file mode 100644 index 000000000000..22960a2631dc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/plan.json @@ -0,0 +1,214 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID BIGINT KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='t1', TIMESTAMP='RTS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_T1 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n T1.F1 F1,\n T1.F2 F2\nFROM S1 S1\nINNER JOIN T1 T1 ON ((S1.ID = T1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_T1", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T1" ], + "sink" : "S1_JOIN_T1", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_T1" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "timestampColumn" : { + "column" : "RTS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "F1 AS T1_F1", "F2 AS T1_F2", "RTS AS T1_RTS", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "T1_F1 AS F1", "T1_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S1_JOIN_T1", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CSAS_S1_JOIN_T1_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/spec.json new file mode 100644 index 000000000000..3ae2ff617a97 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/spec.json @@ -0,0 +1,186 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164292476, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.S1_JOIN_T1" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table join with ts extractor both sides - AVRO", + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 10000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "t1", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 90000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 800000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_T1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "t1", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "RTS", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "TS", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S1_JOIN_T1", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='AVRO');", "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='t1', value_format='AVRO');", "CREATE STREAM S1_JOIN_T1 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, T1.f1, T1.f2 from S1 inner join T1 ON s1.id = t1.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_T1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_T1_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "t1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/topology new file mode 100644 index 000000000000..31c43389bd82 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164292476/topology @@ -0,0 +1,36 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Source: KSTREAM-SOURCE-0000000001 (topics: [t1]) + --> KTABLE-SOURCE-0000000002 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_T1 + <-- Join + Processor: ApplyTimestampTransform-S1_JOIN_T1 (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Project + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Sink: KSTREAM-SINK-0000000011 (topic: S1_JOIN_T1) + <-- ApplyTimestampTransform-S1_JOIN_T1 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/plan.json new file mode 100644 index 000000000000..dec50ca376d2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/plan.json @@ -0,0 +1,214 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID BIGINT KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='t1', TIMESTAMP='RTS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_T1 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n T1.F1 F1,\n T1.F2 F2\nFROM S1 S1\nINNER JOIN T1 T1 ON ((S1.ID = T1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_T1", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T1" ], + "sink" : "S1_JOIN_T1", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_T1" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "timestampColumn" : { + "column" : "RTS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "F1 AS T1_F1", "F2 AS T1_F2", "RTS AS T1_RTS", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "T1_F1 AS F1", "T1_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S1_JOIN_T1", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CSAS_S1_JOIN_T1_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/spec.json new file mode 100644 index 000000000000..7ed7a0b465dd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/spec.json @@ -0,0 +1,150 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164292589, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.S1_JOIN_T1" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table join with ts extractor both sides - JSON", + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 10000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "t1", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 90000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 800000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_T1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "t1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S1_JOIN_T1", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='JSON');", "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='t1', value_format='JSON');", "CREATE STREAM S1_JOIN_T1 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, T1.f1, T1.f2 from S1 inner join T1 ON s1.id = t1.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_T1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_T1_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "t1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/topology new file mode 100644 index 000000000000..31c43389bd82 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164292589/topology @@ -0,0 +1,36 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Source: KSTREAM-SOURCE-0000000001 (topics: [t1]) + --> KTABLE-SOURCE-0000000002 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_T1 + <-- Join + Processor: ApplyTimestampTransform-S1_JOIN_T1 (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Project + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Sink: KSTREAM-SINK-0000000011 (topic: S1_JOIN_T1) + <-- ApplyTimestampTransform-S1_JOIN_T1 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/plan.json new file mode 100644 index 000000000000..e0b66f54f34b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/plan.json @@ -0,0 +1,214 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID BIGINT KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='t1', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_T1 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n T1.F1 F1,\n T1.F2 F2\nFROM S1 S1\nINNER JOIN T1 T1 ON ((S1.ID = T1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_T1", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T1" ], + "sink" : "S1_JOIN_T1", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_T1" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "timestampColumn" : { + "column" : "RTS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "F1 AS T1_F1", "F2 AS T1_F2", "RTS AS T1_RTS", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "T1_F1 AS F1", "T1_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S1_JOIN_T1", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CSAS_S1_JOIN_T1_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/spec.json new file mode 100644 index 000000000000..86abd96a1c58 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/spec.json @@ -0,0 +1,154 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164292694, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.S1_JOIN_T1" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table join with ts extractor both sides - PROTOBUF", + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 10000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "t1", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 90000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 800000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_T1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "t1", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n string F2 = 2;\n int64 RTS = 3;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 TS = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S1_JOIN_T1", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='PROTOBUF');", "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='t1', value_format='PROTOBUF');", "CREATE STREAM S1_JOIN_T1 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, T1.f1, T1.f2 from S1 inner join T1 ON s1.id = t1.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_T1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_T1_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "t1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/topology new file mode 100644 index 000000000000..31c43389bd82 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164292694/topology @@ -0,0 +1,36 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Source: KSTREAM-SOURCE-0000000001 (topics: [t1]) + --> KTABLE-SOURCE-0000000002 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_T1 + <-- Join + Processor: ApplyTimestampTransform-S1_JOIN_T1 (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Project + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Sink: KSTREAM-SINK-0000000011 (topic: S1_JOIN_T1) + <-- ApplyTimestampTransform-S1_JOIN_T1 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/plan.json new file mode 100644 index 000000000000..7f4e12106596 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/plan.json @@ -0,0 +1,201 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/spec.json new file mode 100644 index 000000000000..e3f9a0fdc3db --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/spec.json @@ -0,0 +1,189 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164292808, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join with ts - AVRO", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 19000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 18000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 19000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 19000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 18000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "TS", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='AVRO');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar) WITH (kafka_topic='s2', value_format='AVRO');", "CREATE TABLE S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 ON s1.id = s2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "s2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/topology new file mode 100644 index 000000000000..bab2fc676c8b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.1.0_1594164292808/topology @@ -0,0 +1,51 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [s1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [s2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TOSTREAM-0000000016 + Sink: KSTREAM-SINK-0000000017 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/plan.json new file mode 100644 index 000000000000..2c5990e3bad7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/plan.json @@ -0,0 +1,201 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/spec.json new file mode 100644 index 000000000000..737ddfd06c10 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/spec.json @@ -0,0 +1,157 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164292945, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join with ts - JSON", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 19000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 18000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 19000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 19000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 18000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='JSON');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar) WITH (kafka_topic='s2', value_format='JSON');", "CREATE TABLE S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 ON s1.id = s2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "s2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/topology new file mode 100644 index 000000000000..bab2fc676c8b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.1.0_1594164292945/topology @@ -0,0 +1,51 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [s1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [s2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TOSTREAM-0000000016 + Sink: KSTREAM-SINK-0000000017 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/plan.json new file mode 100644 index 000000000000..513681c7723b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/plan.json @@ -0,0 +1,201 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/spec.json new file mode 100644 index 000000000000..811b0235cea4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/spec.json @@ -0,0 +1,161 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164293090, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join with ts - PROTOBUF", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 19000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 18000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 19000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 19000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 18000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 TS = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n string F2 = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='PROTOBUF');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar) WITH (kafka_topic='s2', value_format='PROTOBUF');", "CREATE TABLE S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 ON s1.id = s2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "s2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/topology new file mode 100644 index 000000000000..bab2fc676c8b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.1.0_1594164293090/topology @@ -0,0 +1,51 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [s1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [s2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TOSTREAM-0000000016 + Sink: KSTREAM-SINK-0000000017 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/plan.json new file mode 100644 index 000000000000..5c2de66741c0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/plan.json @@ -0,0 +1,207 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', TIMESTAMP='RTS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "timestampColumn" : { + "column" : "RTS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/spec.json new file mode 100644 index 000000000000..933e5ab4fe13 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/spec.json @@ -0,0 +1,195 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164293221, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join with ts extractor both sides - AVRO", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "TS", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "RTS", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='AVRO');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='s2', value_format='AVRO');", "CREATE TABLE S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 ON s1.id = s2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "s2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/topology new file mode 100644 index 000000000000..bab2fc676c8b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.1.0_1594164293221/topology @@ -0,0 +1,51 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [s1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [s2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TOSTREAM-0000000016 + Sink: KSTREAM-SINK-0000000017 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/plan.json new file mode 100644 index 000000000000..c28e3e17f393 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/plan.json @@ -0,0 +1,207 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', TIMESTAMP='RTS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "timestampColumn" : { + "column" : "RTS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/spec.json new file mode 100644 index 000000000000..b60b1bb52d6d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/spec.json @@ -0,0 +1,159 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164293356, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join with ts extractor both sides - JSON", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='JSON');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='s2', value_format='JSON');", "CREATE TABLE S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 ON s1.id = s2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "s2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/topology new file mode 100644 index 000000000000..bab2fc676c8b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.1.0_1594164293356/topology @@ -0,0 +1,51 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [s1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [s2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TOSTREAM-0000000016 + Sink: KSTREAM-SINK-0000000017 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/plan.json new file mode 100644 index 000000000000..8d3ff74ec9f7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/plan.json @@ -0,0 +1,207 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS" + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "timestampColumn" : { + "column" : "TS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "timestampColumn" : { + "column" : "RTS" + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS" + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/spec.json new file mode 100644 index 000000000000..d1566c6277db --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/spec.json @@ -0,0 +1,163 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164293507, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join with ts extractor both sides - PROTOBUF", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 TS = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n string F2 = 2;\n int64 RTS = 3;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='PROTOBUF');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='s2', value_format='PROTOBUF');", "CREATE TABLE S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 ON s1.id = s2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "s1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "s2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/topology new file mode 100644 index 000000000000..bab2fc676c8b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.1.0_1594164293507/topology @@ -0,0 +1,51 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [s1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [s2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TOSTREAM-0000000016 + Sink: KSTREAM-SINK-0000000017 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/plan.json new file mode 100644 index 000000000000..3df3625e25d0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT_STREAM (K STRING KEY, SF BIGINT) WITH (KAFKA_TOPIC='stream_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT_STREAM", + "schema" : "`K` STRING KEY, `SF` BIGINT", + "topicName" : "stream_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT_TABLE (ID BIGINT PRIMARY KEY, TF INTEGER) WITH (KAFKA_TOPIC='table_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT_TABLE", + "schema" : "`ID` BIGINT KEY, `TF` INTEGER", + "topicName" : "table_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n *,\n S.ROWTIME S_ROWTIME,\n T.ROWTIME T_ROWTIME\nFROM INPUT_STREAM S\nINNER JOIN INPUT_TABLE T ON ((S.SF = T.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S_SF` BIGINT KEY, `S_K` STRING, `T_ID` BIGINT, `T_TF` INTEGER, `S_ROWTIME` BIGINT, `T_ROWTIME` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT_STREAM", "INPUT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "stream_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `SF` BIGINT" + }, + "keyExpression" : "SF" + }, + "keyColumnNames" : [ "S_SF" ], + "selectExpressions" : [ "SF AS S_SF", "ROWTIME AS S_ROWTIME", "K AS S_K" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "table_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `TF` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "TF AS T_TF", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "keyColName" : "S_SF" + }, + "keyColumnNames" : [ "S_SF" ], + "selectExpressions" : [ "S_K AS S_K", "T_ID AS T_ID", "T_TF AS T_TF", "S_ROWTIME AS S_ROWTIME", "T_ROWTIME AS T_ROWTIME" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/spec.json new file mode 100644 index 000000000000..fc9095a6752d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/spec.json @@ -0,0 +1,135 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164289267, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "on non-STRING value column", + "inputs" : [ { + "topic" : "table_topic", + "key" : 26589, + "value" : { + "TF" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "stream_topic", + "key" : "a", + "value" : { + "SF" : 12589 + }, + "timestamp" : 100 + }, { + "topic" : "table_topic", + "key" : 12589, + "value" : { + "TF" : 12 + }, + "timestamp" : 200 + }, { + "topic" : "stream_topic", + "key" : "b", + "value" : { + "SF" : 12589 + }, + "timestamp" : 300 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 12589, + "value" : { + "S_K" : "b", + "S_ROWTIME" : 300, + "T_ROWTIME" : 300, + "T_ID" : 12589, + "T_TF" : 12 + }, + "timestamp" : 300 + } ], + "topics" : [ { + "name" : "table_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "stream_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT_STREAM (K STRING KEY, SF BIGINT) WITH (kafka_topic='stream_topic', value_format='JSON');", "CREATE TABLE INPUT_TABLE (ID BIGINT PRIMARY KEY, TF INT) WITH (kafka_topic='table_topic', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT *, S.ROWTIME, T.ROWTIME FROM INPUT_STREAM S JOIN INPUT_TABLE T on S.SF = T.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S_SF BIGINT KEY, S_K STRING, T_ID BIGINT, T_TF INT, S_ROWTIME BIGINT, T_ROWTIME BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "stream_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "table_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/topology new file mode 100644 index 000000000000..1e1a18f042ad --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_non-STRING_value_column/6.1.0_1594164289267/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [table_topic]) + --> KTABLE-SOURCE-0000000002 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Join + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [stream_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000006 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasLeft (stores: []) + --> Join-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- PrependAliasLeft + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/plan.json new file mode 100644 index 000000000000..192f82fe9111 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "INNER_JOIN" + }, + "queryId" : "CSAS_INNER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/spec.json new file mode 100644 index 000000000000..5fa29c29d3e1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/spec.json @@ -0,0 +1,201 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286600, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.INNER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table inner join - AVRO", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='test_table', value_format='AVRO');", "CREATE STREAM INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "stream", + "schema" : "T_ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "INNER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_table", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/topology new file mode 100644 index 000000000000..d5d662897385 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_AVRO/6.1.0_1594164286600/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: INNER_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/plan.json new file mode 100644 index 000000000000..0e41c1407cc3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "INNER_JOIN" + }, + "queryId" : "CSAS_INNER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/spec.json new file mode 100644 index 000000000000..df76d197a264 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286698, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.INNER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table inner join - JSON", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='test_table', value_format='JSON');", "CREATE STREAM INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "stream", + "schema" : "T_ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "INNER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_table", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/topology new file mode 100644 index 000000000000..d5d662897385 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_JSON/6.1.0_1594164286698/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: INNER_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/plan.json new file mode 100644 index 000000000000..3af153897d5b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "INNER_JOIN" + }, + "queryId" : "CSAS_INNER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/spec.json new file mode 100644 index 000000000000..46109f2ea886 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/spec.json @@ -0,0 +1,173 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286793, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_INNER_JOIN_0.INNER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table inner join - PROTOBUF", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='test_table', value_format='PROTOBUF');", "CREATE STREAM INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "stream", + "schema" : "T_ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "INNER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_table", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/topology new file mode 100644 index 000000000000..d5d662897385 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF/6.1.0_1594164286793/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: INNER_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/plan.json new file mode 100644 index 000000000000..fe5b31c6e15f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM LEFT_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "LEFT_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "LEFT_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "LEFT_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "LEFT_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "LEFT_JOIN" + }, + "queryId" : "CSAS_LEFT_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/spec.json new file mode 100644 index 000000000000..037802908b05 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/spec.json @@ -0,0 +1,211 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286326, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_LEFT_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.LEFT_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table left join - AVRO", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "LEFT_JOIN", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : null, + "F2" : null + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "LEFT_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='test_table', value_format='AVRO');", "CREATE STREAM LEFT_JOIN as SELECT t.id, name, value, f1, f2 FROM test t left join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_JOIN", + "type" : "stream", + "schema" : "T_ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "LEFT_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_LEFT_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "test_table", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/topology new file mode 100644 index 000000000000..c55b202f1185 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_AVRO/6.1.0_1594164286326/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: LEFT_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/plan.json new file mode 100644 index 000000000000..a267b73033b9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM LEFT_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "LEFT_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "LEFT_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "LEFT_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "LEFT_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "LEFT_JOIN" + }, + "queryId" : "CSAS_LEFT_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/spec.json new file mode 100644 index 000000000000..78345b98d488 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/spec.json @@ -0,0 +1,179 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286416, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_LEFT_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.LEFT_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table left join - JSON", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "LEFT_JOIN", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : null, + "F2" : null + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "LEFT_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='test_table', value_format='JSON');", "CREATE STREAM LEFT_JOIN as SELECT t.id, name, value, f1, f2 FROM test t left join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_JOIN", + "type" : "stream", + "schema" : "T_ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "LEFT_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_LEFT_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_table", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/topology new file mode 100644 index 000000000000..c55b202f1185 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_JSON/6.1.0_1594164286416/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: LEFT_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/plan.json new file mode 100644 index 000000000000..7986b1c33413 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM LEFT_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "LEFT_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "LEFT_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "LEFT_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "LEFT_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "LEFT_JOIN" + }, + "queryId" : "CSAS_LEFT_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/spec.json new file mode 100644 index 000000000000..ef47814f1e75 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/spec.json @@ -0,0 +1,183 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286504, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_LEFT_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_LEFT_JOIN_0.LEFT_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream table left join - PROTOBUF", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "LEFT_JOIN", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "LEFT_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='test_table', value_format='PROTOBUF');", "CREATE STREAM LEFT_JOIN as SELECT t.id, name, value, f1, f2 FROM test t left join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_JOIN", + "type" : "stream", + "schema" : "T_ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "LEFT_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_LEFT_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "test_table", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/topology new file mode 100644 index 000000000000..c55b202f1185 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF/6.1.0_1594164286504/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: LEFT_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/plan.json new file mode 100644 index 000000000000..6d957c706868 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/plan.json @@ -0,0 +1,200 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S (ID BIGINT KEY, NAME STRING) WITH (KAFKA_TOPIC='S', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "S", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S.ID S_ID,\n S.NAME NAME1,\n T.NAME NAME2\nFROM S S\nINNER JOIN T T ON ((S.ID = T.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S_ID` BIGINT KEY, `NAME1` STRING, `NAME2` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S", "T" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "S", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "NAME AS S_NAME", "ROWTIME AS S_ROWTIME", "ID AS S_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "keyColName" : "S_ID" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "S_NAME AS NAME1", "T_NAME AS NAME2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/spec.json new file mode 100644 index 000000000000..71ae092e5238 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/spec.json @@ -0,0 +1,111 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164287813, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "VARCHAR", + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : "VARCHAR", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream to table unwrapped single field value schema on inputs", + "inputs" : [ { + "topic" : "T", + "key" : 0, + "value" : "b", + "timestamp" : 0 + }, { + "topic" : "S", + "key" : 0, + "value" : "a", + "timestamp" : 10 + }, { + "topic" : "S", + "key" : 0, + "value" : null, + "timestamp" : 20 + }, { + "topic" : "T", + "key" : 0, + "value" : null, + "timestamp" : 30 + }, { + "topic" : "S", + "key" : 0, + "value" : null, + "timestamp" : 40 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : "a", + "NAME2" : "b" + }, + "timestamp" : 10 + } ], + "topics" : [ { + "name" : "S", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S (ID BIGINT KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='S', value_format='JSON');", "CREATE TABLE T (ID BIGINT PRIMARY KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='T', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT s.id, s.name name1, t.name name2 FROM S JOIN T ON S.id = T.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "S", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/topology new file mode 100644 index 000000000000..695e75836168 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164287813/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [S]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [T]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/plan.json new file mode 100644 index 000000000000..839c40c78efe --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/plan.json @@ -0,0 +1,202 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S (ID BIGINT KEY, NAME STRING) WITH (KAFKA_TOPIC='S', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "S", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT WITH (WRAP_SINGLE_VALUE=false) AS SELECT\n S.ID S_ID,\n S.NAME NAME\nFROM S S\nINNER JOIN T T ON ((S.ID = T.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S_ID` BIGINT KEY, `NAME` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S", "T" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "S", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "NAME AS S_NAME", "ROWTIME AS S_ROWTIME", "ID AS S_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "keyColName" : "S_ID" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "S_NAME AS NAME" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/spec.json new file mode 100644 index 000000000000..18af7bdc1c20 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/spec.json @@ -0,0 +1,108 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164287892, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "VARCHAR", + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : "VARCHAR", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "VARCHAR" + }, + "testCase" : { + "name" : "stream to table unwrapped single field value schema on inputs and output", + "inputs" : [ { + "topic" : "T", + "key" : 0, + "value" : "b", + "timestamp" : 0 + }, { + "topic" : "S", + "key" : 0, + "value" : "a", + "timestamp" : 10 + }, { + "topic" : "S", + "key" : 0, + "value" : null, + "timestamp" : 20 + }, { + "topic" : "T", + "key" : 0, + "value" : null, + "timestamp" : 30 + }, { + "topic" : "S", + "key" : 0, + "value" : null, + "timestamp" : 40 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : "a", + "timestamp" : 10 + } ], + "topics" : [ { + "name" : "S", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S (ID BIGINT KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='S', value_format='JSON');", "CREATE TABLE T (ID BIGINT PRIMARY KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='T', value_format='JSON');", "CREATE STREAM OUTPUT WITH (WRAP_SINGLE_VALUE=false) AS SELECT s.id, s.name name FROM S JOIN T ON S.id = T.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "S", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/topology new file mode 100644 index 000000000000..695e75836168 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164287892/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [S]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [T]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/plan.json new file mode 100644 index 000000000000..b9a528b00949 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S (ID BIGINT KEY, NAME STRING) WITH (KAFKA_TOPIC='S', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "S", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S.ID S_ID,\n S.NAME NAME1,\n T.NAME NAME2\nFROM S S\nINNER JOIN T T ON ((S.ID = T.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S_ID` BIGINT KEY, `NAME1` STRING, `NAME2` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S", "T" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "S", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "NAME AS S_NAME", "ROWTIME AS S_ROWTIME", "ID AS S_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "keyColName" : "S_ID" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "S_NAME AS NAME1", "T_NAME AS NAME2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/spec.json new file mode 100644 index 000000000000..6309d9d4ee4d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164287711, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream to table wrapped single field value schema on inputs", + "inputs" : [ { + "topic" : "T", + "key" : 0, + "value" : { + "NAME" : "b" + }, + "timestamp" : 0 + }, { + "topic" : "S", + "key" : 0, + "value" : { + "NAME" : "a" + }, + "timestamp" : 10 + }, { + "topic" : "S", + "key" : 0, + "value" : { + "NAME" : null + }, + "timestamp" : 20 + }, { + "topic" : "T", + "key" : 0, + "value" : { + "NAME" : null + }, + "timestamp" : 30 + }, { + "topic" : "S", + "key" : 0, + "value" : { + "NAME" : null + }, + "timestamp" : 40 + }, { + "topic" : "T", + "key" : 0, + "value" : null, + "timestamp" : 50 + }, { + "topic" : "S", + "key" : 0, + "value" : { + "NAME" : "a" + }, + "timestamp" : 60 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : "a", + "NAME2" : "b" + }, + "timestamp" : 10 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : null, + "NAME2" : "b" + }, + "timestamp" : 20 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : null, + "NAME2" : null + }, + "timestamp" : 40 + } ], + "topics" : [ { + "name" : "S", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S (ID BIGINT KEY, NAME STRING) WITH (kafka_topic='S', value_format='JSON');", "CREATE TABLE T (ID BIGINT PRIMARY KEY, NAME STRING) WITH (kafka_topic='T', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT s.id, s.name name1, t.name name2 FROM S JOIN T ON S.id = T.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "S", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/topology new file mode 100644 index 000000000000..695e75836168 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287711/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [S]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [T]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/plan.json new file mode 100644 index 000000000000..0b5a88120d90 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM L (A INTEGER, B INTEGER, C INTEGER) WITH (KAFKA_TOPIC='LEFT', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "L", + "schema" : "`A` INTEGER, `B` INTEGER, `C` INTEGER", + "topicName" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE R (A INTEGER PRIMARY KEY, B INTEGER, C INTEGER) WITH (KAFKA_TOPIC='RIGHT', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "R", + "schema" : "`A` INTEGER KEY, `B` INTEGER, `C` INTEGER", + "topicName" : "RIGHT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM L L\nINNER JOIN R R ON ((L.A = R.A))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`L_A` INTEGER KEY, `L_B` INTEGER, `L_C` INTEGER, `R_A` INTEGER, `R_B` INTEGER, `R_C` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`A` INTEGER, `B` INTEGER, `C` INTEGER" + }, + "keyExpression" : "A" + }, + "keyColumnNames" : [ "L_A" ], + "selectExpressions" : [ "A AS L_A", "B AS L_B", "C AS L_C", "ROWTIME AS L_ROWTIME" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "RIGHT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`A` INTEGER KEY, `B` INTEGER, `C` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "R_A" ], + "selectExpressions" : [ "B AS R_B", "C AS R_C", "ROWTIME AS R_ROWTIME", "A AS R_A" ] + }, + "keyColName" : "L_A" + }, + "keyColumnNames" : [ "L_A" ], + "selectExpressions" : [ "L_B AS L_B", "L_C AS L_C", "R_A AS R_A", "R_B AS R_B", "R_C AS R_C" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/spec.json new file mode 100644 index 000000000000..9b112bd0c833 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/spec.json @@ -0,0 +1,121 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164291624, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "streams with no key columns (stream->table)", + "inputs" : [ { + "topic" : "RIGHT", + "key" : 0, + "value" : { + "B" : -1, + "C" : -2 + }, + "timestamp" : 10 + }, { + "topic" : "LEFT", + "key" : "ignored", + "value" : { + "A" : 0, + "B" : 1, + "C" : 2 + }, + "timestamp" : 11 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "R_A" : 0, + "L_B" : 1, + "R_B" : -1, + "L_C" : 2, + "R_C" : -2 + }, + "timestamp" : 11 + } ], + "topics" : [ { + "name" : "LEFT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "RIGHT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM L (A INT, B INT, C INT) WITH (kafka_topic='LEFT', value_format='JSON');", "CREATE TABLE R (A INT PRIMARY KEY, B INT, C INT) WITH (kafka_topic='RIGHT', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT * FROM L INNER JOIN R ON L.A = R.A;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "L_A INT KEY, L_B INT, L_C INT, R_A INT, R_B INT, R_C INT" + } ], + "topics" : { + "topics" : [ { + "name" : "LEFT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "RIGHT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/topology new file mode 100644 index 000000000000..b2089e8a21be --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_streams_with_no_key_columns_(stream-_table)/6.1.0_1594164291624/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [RIGHT]) + --> KTABLE-SOURCE-0000000002 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Join + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [LEFT]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000006 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasLeft (stores: []) + --> Join-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- PrependAliasLeft + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/plan.json new file mode 100644 index 000000000000..deb7315d959e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/plan.json @@ -0,0 +1,311 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE_2 (ID BIGINT PRIMARY KEY, F3 STRING) WITH (KAFKA_TOPIC='right_topic_2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE_2", + "schema" : "`ID` BIGINT KEY, `F3` STRING", + "topicName" : "right_topic_2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INNER_JOIN WITH (PARTITIONS=4) AS SELECT\n T.ID ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INNER_JOIN", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "INNER_JOIN" + }, + "queryId" : "CTAS_INNER_JOIN_0" + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INNER_JOIN_2 AS SELECT\n TT.ID TT_ID,\n TT.NAME NAME,\n TT.F1 F1,\n T.F3 F3\nFROM INNER_JOIN TT\nINNER JOIN TEST_TABLE_2 T ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INNER_JOIN_2", + "schema" : "`TT_ID` BIGINT KEY, `NAME` STRING, `F1` STRING, `F3` STRING", + "topicName" : "INNER_JOIN_2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INNER_JOIN", "TEST_TABLE_2" ], + "sink" : "INNER_JOIN_2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN_2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "NAME AS TT_NAME", "VALUE AS TT_VALUE", "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic_2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F3` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "F3 AS T_F3", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "keyColName" : "TT_ID" + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "TT_NAME AS NAME", "TT_F1 AS F1", "T_F3 AS F3" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "INNER_JOIN_2" + }, + "queryId" : "CTAS_INNER_JOIN_2_1" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/spec.json new file mode 100644 index 000000000000..9e41a9b48677 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286911, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_INNER_JOIN_2_1.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_2_1.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_2_1.INNER_JOIN_2" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table join pipeline - JSON", + "inputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "X", + "VALUE" : 0, + "F1" : "yo dawg", + "F2" : 50 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic_2", + "key" : 0, + "value" : { + "F3" : "I heard you like joins" + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 100, + "value" : { + "NAME" : "X", + "VALUE" : 0, + "F1" : "KSQL has table-table joins", + "F2" : 50 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic_2", + "key" : 100, + "value" : { + "F3" : "so now you can join your join" + }, + "timestamp" : 20000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN_2", + "key" : 0, + "value" : { + "NAME" : "X", + "F1" : "yo dawg", + "F3" : "I heard you like joins" + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN_2", + "key" : 100, + "value" : { + "NAME" : "X", + "F1" : "KSQL has table-table joins", + "F3" : "so now you can join your join" + }, + "timestamp" : 20000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right_topic_2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN_2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE_2 (ID BIGINT PRIMARY KEY, F3 varchar) WITH (kafka_topic='right_topic_2', value_format='JSON');", "CREATE TABLE INNER_JOIN WITH (PARTITIONS=4) as SELECT t.id AS ID, name, value, f1, f2 FROM test t join TEST_TABLE tt on t.id = tt.id;", "CREATE TABLE INNER_JOIN_2 AS SELECT tt.id, name, f1, f3 FROM inner_join tt join TEST_TABLE_2 t ON t.id = tt.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "INNER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "INNER_JOIN_2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_2_1-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_2_1-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic_2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/topology new file mode 100644 index 000000000000..0e4b0f7e92de --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_join_pipeline_-_JSON/6.1.0_1594164286911/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [INNER_JOIN]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic_2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: INNER_JOIN_2) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/plan.json new file mode 100644 index 000000000000..0c3412c21206 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "INNER_JOIN" + }, + "queryId" : "CTAS_INNER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/spec.json new file mode 100644 index 000000000000..cf8cff57b66d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/spec.json @@ -0,0 +1,228 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164285187, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_INNER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_0.INNER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join - AVRO", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "right_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='AVRO');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='AVRO');", "CREATE TABLE INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "table", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "INNER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/topology new file mode 100644 index 000000000000..e2e1d1b38d22 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_AVRO/6.1.0_1594164285187/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: INNER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/plan.json new file mode 100644 index 000000000000..14289f6e7195 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "INNER_JOIN" + }, + "queryId" : "CTAS_INNER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/spec.json new file mode 100644 index 000000000000..4dbd1742e295 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/spec.json @@ -0,0 +1,196 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164285316, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_INNER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_0.INNER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join - JSON", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "table", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "INNER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/topology new file mode 100644 index 000000000000..e2e1d1b38d22 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_JSON/6.1.0_1594164285316/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: INNER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/plan.json new file mode 100644 index 000000000000..916833626e17 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "INNER_JOIN" + }, + "queryId" : "CTAS_INNER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/spec.json new file mode 100644 index 000000000000..1b05c32a6317 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/spec.json @@ -0,0 +1,200 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164285447, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_INNER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_INNER_JOIN_0.INNER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table inner join - PROTOBUF", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "right_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", "CREATE TABLE INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "table", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "INNER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/topology new file mode 100644 index 000000000000..e2e1d1b38d22 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF/6.1.0_1594164285447/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: INNER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/plan.json new file mode 100644 index 000000000000..f4e067d34cc4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/plan.json @@ -0,0 +1,196 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nWHERE ((T.VALUE > 10) AND (TT.F2 > 5))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `F1` STRING, `F2` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "WhereFilter" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "filterExpression" : "((T_VALUE > 10) AND (TT_F2 > 5))" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/spec.json new file mode 100644 index 000000000000..c44b0afdfae0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/spec.json @@ -0,0 +1,195 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164287151, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table join with where clause", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "right_topic", + "key" : 90, + "value" : { + "F1" : "b", + "F2" : 10 + }, + "timestamp" : 18000 + }, { + "topic" : "right_topic", + "key" : 90, + "value" : null, + "timestamp" : 19000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "F1" : "b", + "F2" : 10 + }, + "timestamp" : 18000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : null, + "timestamp" : 19000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT t.id, name, tt.f1, f2 FROM test t JOIN test_table tt ON t.id = tt.id WHERE t.value > 10 AND tt.f2 > 5;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "T_ID BIGINT KEY, `NAME` STRING, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/topology new file mode 100644 index 000000000000..f603ab5711b0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_join_with_where_clause/6.1.0_1594164287151/topology @@ -0,0 +1,57 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> WhereFilter-ApplyPredicate + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: WhereFilter-ApplyPredicate (stores: []) + --> WhereFilter-Filter + <-- KTABLE-MERGE-0000000012 + Processor: WhereFilter-Filter (stores: []) + --> WhereFilter-PostProcess + <-- WhereFilter-ApplyPredicate + Processor: WhereFilter-PostProcess (stores: []) + --> Project + <-- WhereFilter-Filter + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000019 + <-- WhereFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000019 (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Project + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/plan.json new file mode 100644 index 000000000000..52ad89cc5a7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/spec.json new file mode 100644 index 000000000000..937aea726213 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/spec.json @@ -0,0 +1,250 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164284630, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table left join - AVRO", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : null, + "F2" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : null, + "F2" : null + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : null, + "F2" : null + }, + "timestamp" : 17000 + } ], + "topics" : [ { + "name" : "right_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='AVRO');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='AVRO');", "CREATE TABLE OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/topology new file mode 100644 index 000000000000..3ab6556fbea7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_AVRO/6.1.0_1594164284630/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/plan.json new file mode 100644 index 000000000000..5b60d416b54d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/spec.json new file mode 100644 index 000000000000..0ef18fa4dd7d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/spec.json @@ -0,0 +1,218 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164284817, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table left join - JSON", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : null, + "F2" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : null, + "F2" : null + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : null, + "F2" : null + }, + "timestamp" : 17000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/topology new file mode 100644 index 000000000000..3ab6556fbea7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_JSON/6.1.0_1594164284817/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/plan.json new file mode 100644 index 000000000000..07f1c289442d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/spec.json new file mode 100644 index 000000000000..2c482fbbbfc3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/spec.json @@ -0,0 +1,222 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164285011, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table left join - PROTOBUF", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + } ], + "topics" : [ { + "name" : "right_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", "CREATE TABLE OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/topology new file mode 100644 index 000000000000..3ab6556fbea7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF/6.1.0_1594164285011/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/plan.json new file mode 100644 index 000000000000..586ea988ce28 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTER_JOIN AS SELECT\n ROWKEY ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nFULL OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTER_JOIN", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "OUTER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTER_JOIN" + }, + "queryId" : "CTAS_OUTER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/spec.json new file mode 100644 index 000000000000..779b90970a8f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/spec.json @@ -0,0 +1,250 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164285739, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTER_JOIN_0.OUTER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table outer join - AVRO", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + } ], + "outputs" : [ { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : null, + "F2" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTER_JOIN", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : null, + "F2" : null + }, + "timestamp" : 11000 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTER_JOIN", + "key" : 15, + "value" : { + "NAME" : null, + "VALUE" : null, + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "OUTER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "schema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "format" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='AVRO');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='AVRO');", "CREATE TABLE OUTER_JOIN as SELECT ROWKEY AS ID, name, value, f1, f2 FROM test t FULL OUTER join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTER_JOIN", + "type" : "table", + "schema" : "ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/topology new file mode 100644 index 000000000000..5508ffbe4db2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_AVRO/6.1.0_1594164285739/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: [right_topic-STATE-STORE-0000000006]) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/plan.json new file mode 100644 index 000000000000..717507b56dd4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTER_JOIN AS SELECT\n ROWKEY ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nFULL OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTER_JOIN", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "OUTER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTER_JOIN" + }, + "queryId" : "CTAS_OUTER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/spec.json new file mode 100644 index 000000000000..c75fc757a4be --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/spec.json @@ -0,0 +1,218 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164285943, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTER_JOIN_0.OUTER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table outer join - JSON", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + } ], + "outputs" : [ { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : null, + "F2" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTER_JOIN", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : null, + "F2" : null + }, + "timestamp" : 11000 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTER_JOIN", + "key" : 15, + "value" : { + "NAME" : null, + "VALUE" : null, + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "OUTER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "OUTER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE OUTER_JOIN as SELECT ROWKEY AS ID, name, value, f1, f2 FROM test t FULL OUTER join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTER_JOIN", + "type" : "table", + "schema" : "ID BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/topology new file mode 100644 index 000000000000..5508ffbe4db2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_JSON/6.1.0_1594164285943/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: [right_topic-STATE-STORE-0000000006]) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/plan.json new file mode 100644 index 000000000000..a82340edf793 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTER_JOIN AS SELECT\n ROWKEY ID,\n T.ID T_ID,\n TT.ID TT_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nFULL OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTER_JOIN", + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "topicName" : "OUTER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T_ID AS T_ID", "TT_ID AS TT_ID", "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "OUTER_JOIN" + }, + "queryId" : "CTAS_OUTER_JOIN_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/spec.json new file mode 100644 index 000000000000..ffcf8ffbf822 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/spec.json @@ -0,0 +1,236 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164286127, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTER_JOIN_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTER_JOIN_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTER_JOIN_0.OUTER_JOIN" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table table outer join - PROTOBUF", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + } ], + "outputs" : [ { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 0, + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTER_JOIN", + "key" : 10, + "value" : { + "T_ID" : 10, + "TT_ID" : 0, + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTER_JOIN", + "key" : 15, + "value" : { + "T_ID" : 0, + "TT_ID" : 15, + "NAME" : "", + "VALUE" : 0, + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "OUTER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "schema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "format" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", "CREATE TABLE OUTER_JOIN as SELECT ROWKEY AS ID, t.id, tt.id, name, value, f1, f2 FROM test t FULL OUTER join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTER_JOIN", + "type" : "table", + "schema" : "ID BIGINT KEY, T_ID BIGINT, TT_ID BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTER_JOIN", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/topology new file mode 100644 index 000000000000..5508ffbe4db2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF/6.1.0_1594164286127/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: [right_topic-STATE-STORE-0000000006]) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/plan.json new file mode 100644 index 000000000000..35e3a06c09c7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/plan.json @@ -0,0 +1,193 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T1', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T2', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n T1.NAME NAME1,\n T2.NAME NAME2\nFROM T1 T1\nINNER JOIN T2 T2 ON ((T1.ID = T2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `NAME1` STRING, `NAME2` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "NAME AS T1_NAME", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "NAME AS T2_NAME", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_NAME AS NAME1", "T2_NAME AS NAME2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/spec.json new file mode 100644 index 000000000000..a2a0c0d5478a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/spec.json @@ -0,0 +1,125 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164288094, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "VARCHAR", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "VARCHAR", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table to table unwrapped single field value schema on inputs", + "inputs" : [ { + "topic" : "T1", + "key" : 0, + "value" : "a", + "timestamp" : 0 + }, { + "topic" : "T2", + "key" : 0, + "value" : "b", + "timestamp" : 10 + }, { + "topic" : "T1", + "key" : 0, + "value" : null, + "timestamp" : 20 + }, { + "topic" : "T2", + "key" : 0, + "value" : null, + "timestamp" : 30 + }, { + "topic" : "T1", + "key" : 0, + "value" : null, + "timestamp" : 40 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : "a", + "NAME2" : "b" + }, + "timestamp" : 10 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 20 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='T1', value_format='JSON');", "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='T2', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT t1.id, t1.name name1, t2.name name2 FROM T1 JOIN T2 ON T1.id = T2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/topology new file mode 100644 index 000000000000..a9dcebedb74f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs/6.1.0_1594164288094/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [T1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [T2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/plan.json new file mode 100644 index 000000000000..882bc261770e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/plan.json @@ -0,0 +1,195 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T1', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T2', VALUE_FORMAT='JSON', WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT WITH (WRAP_SINGLE_VALUE=false) AS SELECT\n T1.ID T1_ID,\n T1.NAME NAME\nFROM T1 T1\nINNER JOIN T2 T2 ON ((T1.ID = T2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `NAME` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "NAME AS T1_NAME", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "NAME AS T2_NAME", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_NAME AS NAME" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "options" : [ "UNWRAP_SINGLE_VALUES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/spec.json new file mode 100644 index 000000000000..84cafed00a40 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/spec.json @@ -0,0 +1,122 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164288199, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "VARCHAR", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "VARCHAR", + "CTAS_OUTPUT_0.OUTPUT" : "VARCHAR" + }, + "testCase" : { + "name" : "table to table unwrapped single field value schema on inputs and output", + "inputs" : [ { + "topic" : "T1", + "key" : 0, + "value" : "a", + "timestamp" : 0 + }, { + "topic" : "T2", + "key" : 0, + "value" : "b", + "timestamp" : 10 + }, { + "topic" : "T1", + "key" : 0, + "value" : null, + "timestamp" : 20 + }, { + "topic" : "T2", + "key" : 0, + "value" : null, + "timestamp" : 30 + }, { + "topic" : "T1", + "key" : 0, + "value" : null, + "timestamp" : 40 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : "a", + "timestamp" : 10 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 20 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='T1', value_format='JSON');", "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (WRAP_SINGLE_VALUE=false, kafka_topic='T2', value_format='JSON');", "CREATE TABLE OUTPUT WITH (WRAP_SINGLE_VALUE=false) AS SELECT t1.id, t1.name name FROM T1 JOIN T2 ON T1.id = T2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/topology new file mode 100644 index 000000000000..a9dcebedb74f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_unwrapped_single_field_value_schema_on_inputs_and_output/6.1.0_1594164288199/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [T1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [T2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/plan.json new file mode 100644 index 000000000000..c76a35fd676d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T1', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='T2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n T1.NAME NAME1,\n T2.NAME NAME2\nFROM T1 T1\nINNER JOIN T2 T2 ON ((T1.ID = T2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `NAME1` STRING, `NAME2` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "NAME AS T1_NAME", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "NAME AS T2_NAME", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_NAME AS NAME1", "T2_NAME AS NAME2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/spec.json new file mode 100644 index 000000000000..ca11d23d9f27 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164287985, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table to table wrapped single field value schema on inputs", + "inputs" : [ { + "topic" : "T1", + "key" : 0, + "value" : { + "NAME" : "a" + }, + "timestamp" : 0 + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "NAME" : "b" + }, + "timestamp" : 10 + }, { + "topic" : "T1", + "key" : 0, + "value" : { + "NAME" : null + }, + "timestamp" : 20 + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "NAME" : null + }, + "timestamp" : 30 + }, { + "topic" : "T1", + "key" : 0, + "value" : { + "NAME" : null + }, + "timestamp" : 40 + }, { + "topic" : "T1", + "key" : 0, + "value" : null, + "timestamp" : 50 + }, { + "topic" : "T2", + "key" : 0, + "value" : null, + "timestamp" : 60 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : "a", + "NAME2" : "b" + }, + "timestamp" : 10 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : null, + "NAME2" : "b" + }, + "timestamp" : 20 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : null, + "NAME2" : null + }, + "timestamp" : 30 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME1" : null, + "NAME2" : null + }, + "timestamp" : 40 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 50 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (kafka_topic='T1', value_format='JSON');", "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (kafka_topic='T2', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT t1.id, t1.name name1, t2.name name2 FROM T1 JOIN T2 ON T1.id = T2.id;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T1", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "T2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/topology new file mode 100644 index 000000000000..a9dcebedb74f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_to_table_wrapped_single_field_value_schema_on_inputs/6.1.0_1594164287985/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [T1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [T2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/plan.json new file mode 100644 index 000000000000..1583390f7c27 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE L (ID INTEGER PRIMARY KEY, V0 INTEGER, V1 INTEGER) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "L", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE R (ID INTEGER PRIMARY KEY, V0 INTEGER, V1 INTEGER) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "R", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n ROWKEY ROWKEY,\n L.ID L_ID,\n R.ID R_ID,\n L.V0 L_V0,\n R.V1 R_V1\nFROM L L\nFULL OUTER JOIN R R ON ((L.ID = R.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` INTEGER KEY, `L_ID` INTEGER, `R_ID` INTEGER, `L_V0` INTEGER, `R_V1` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "V0 AS L_V0", "V1 AS L_V1", "ROWTIME AS L_ROWTIME", "ID AS L_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "R_ID" ], + "selectExpressions" : [ "V0 AS R_V0", "V1 AS R_V1", "ROWTIME AS R_ROWTIME", "ID AS R_ID" ] + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "L_ID AS L_ID", "R_ID AS R_ID", "L_V0 AS L_V0", "R_V1 AS R_V1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/spec.json new file mode 100644 index 000000000000..6e9eab2c7948 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/spec.json @@ -0,0 +1,128 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164279482, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "unaliased synthetic join key", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "V0" : 2, + "V1" : 3 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "V0" : 4, + "V1" : 5 + }, + "timestamp" : 100 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "L_ID" : 1, + "R_ID" : null, + "L_V0" : 2, + "R_V1" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "L_ID" : 1, + "R_ID" : 1, + "L_V0" : 2, + "R_V1" : 5 + }, + "timestamp" : 100 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE L (ID INT PRIMARY KEY, V0 INT, V1 INT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE R (ID INT PRIMARY KEY, V0 INT, V1 INT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT ROWKEY, L.ID, R.ID, L.V0, R.V1 FROM L FULL OUTER JOIN R on L.id = R.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "ROWKEY INT KEY, L_ID INT, R_ID INT, L_V0 INT, R_V1 INT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/topology new file mode 100644 index 000000000000..60e94fef40ea --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_unaliased_synthetic_join_key/6.1.0_1594164279482/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: [right_topic-STATE-STORE-0000000006]) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/plan.json new file mode 100644 index 000000000000..7c781739312f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE L (ROWKEY INTEGER PRIMARY KEY, ROWKEY_1 INTEGER, ROWKEY_2 INTEGER) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "L", + "schema" : "`ROWKEY` INTEGER KEY, `ROWKEY_1` INTEGER, `ROWKEY_2` INTEGER", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE R (ROWKEY_3 INTEGER PRIMARY KEY, ROWKEY_4 INTEGER, ROWKEY_5 INTEGER) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "R", + "schema" : "`ROWKEY_3` INTEGER KEY, `ROWKEY_4` INTEGER, `ROWKEY_5` INTEGER", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n ROWKEY_6 ROWKEY_6,\n L.ROWKEY ROWKEY,\n R.ROWKEY_3 ROWKEY_3,\n L.ROWKEY_1 ROWKEY_1,\n R.ROWKEY_5 ROWKEY_5\nFROM L L\nFULL OUTER JOIN R R ON ((L.ROWKEY = R.ROWKEY_3))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY_6` INTEGER KEY, `ROWKEY` INTEGER, `ROWKEY_3` INTEGER, `ROWKEY_1` INTEGER, `ROWKEY_5` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `ROWKEY_1` INTEGER, `ROWKEY_2` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "L_ROWKEY" ], + "selectExpressions" : [ "ROWKEY_1 AS L_ROWKEY_1", "ROWKEY_2 AS L_ROWKEY_2", "ROWTIME AS L_ROWTIME", "ROWKEY AS L_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ROWKEY_3` INTEGER KEY, `ROWKEY_4` INTEGER, `ROWKEY_5` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "R_ROWKEY_3" ], + "selectExpressions" : [ "ROWKEY_4 AS R_ROWKEY_4", "ROWKEY_5 AS R_ROWKEY_5", "ROWTIME AS R_ROWTIME", "ROWKEY_3 AS R_ROWKEY_3" ] + }, + "keyColName" : "ROWKEY_6" + }, + "keyColumnNames" : [ "ROWKEY_6" ], + "selectExpressions" : [ "L_ROWKEY AS ROWKEY", "R_ROWKEY_3 AS ROWKEY_3", "L_ROWKEY_1 AS ROWKEY_1", "R_ROWKEY_5 AS ROWKEY_5" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/spec.json new file mode 100644 index 000000000000..b31110135a7d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/spec.json @@ -0,0 +1,128 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164288627, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "with generated column name clashes", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "ROWKEY_1" : 2, + "ROWKEY_2" : 3 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "ROWKEY_4" : 4, + "ROWKEY_5" : 5 + }, + "timestamp" : 100 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ROWKEY" : 1, + "ROWKEY_3" : null, + "ROWKEY_1" : 2, + "ROWKEY_5" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ROWKEY" : 1, + "ROWKEY_3" : 1, + "ROWKEY_1" : 2, + "ROWKEY_5" : 5 + }, + "timestamp" : 100 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE L (ROWKEY INT PRIMARY KEY, ROWKEY_1 INT, ROWKEY_2 INT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE R (ROWKEY_3 INT PRIMARY KEY, ROWKEY_4 INT, ROWKEY_5 INT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT ROWKEY_6, L.ROWKEY, R.ROWKEY_3, L.ROWKEY_1, R.ROWKEY_5 FROM L FULL OUTER JOIN R on L.ROWKEY = R.ROWKEY_3;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "ROWKEY_6 INT KEY, ROWKEY INT, ROWKEY_3 INT, ROWKEY_1 INT, ROWKEY_5 INT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/topology new file mode 100644 index 000000000000..60e94fef40ea --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_with_generated_column_name_clashes/6.1.0_1594164288627/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left_topic-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: [right_topic-STATE-STORE-0000000006]) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/plan.json new file mode 100644 index 000000000000..045543e40bd3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/plan.json @@ -0,0 +1,130 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K BIGINT PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='input', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` BIGINT KEY, `ID` BIGINT", + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.ID ID,\n AS_VALUE(INPUT.K) KEY\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` BIGINT KEY, `ID` BIGINT, `KEY` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` BIGINT KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "AS_VALUE(K) AS KEY" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/spec.json new file mode 100644 index 000000000000..5d471fcc386f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/spec.json @@ -0,0 +1,108 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164294074, + "path" : "query-validation-tests/key-schemas.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table explicit KAFKA BIGINT KEY", + "inputs" : [ { + "topic" : "input", + "key" : 3, + "value" : { + "id" : 1 + } + }, { + "topic" : "input", + "key" : 2, + "value" : { + "id" : 2 + } + }, { + "topic" : "input", + "key" : 1, + "value" : { + "id" : 3 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "ID" : 1, + "KEY" : 3 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "ID" : 2, + "KEY" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 3, + "KEY" : 1 + } + } ], + "topics" : [ { + "name" : "input", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K BIGINT PRIMARY KEY, ID bigint) WITH (kafka_topic='input',value_format='JSON');", "CREATE TABLE OUTPUT as SELECT K, ID, AS_VALUE(K) as KEY FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "K BIGINT KEY, ID BIGINT, KEY BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "input", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/topology new file mode 100644 index 000000000000..f15bde0bc817 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_BIGINT_KEY/6.1.0_1594164294074/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/plan.json new file mode 100644 index 000000000000..955cfe63f328 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/plan.json @@ -0,0 +1,130 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K DOUBLE PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='input', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` DOUBLE KEY, `ID` BIGINT", + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.ID ID,\n AS_VALUE(INPUT.K) KEY\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` DOUBLE KEY, `ID` BIGINT, `KEY` DOUBLE", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` DOUBLE KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "AS_VALUE(K) AS KEY" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/spec.json new file mode 100644 index 000000000000..3686d1794056 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/spec.json @@ -0,0 +1,108 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164294154, + "path" : "query-validation-tests/key-schemas.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table explicit KAFKA DOUBLE KEY", + "inputs" : [ { + "topic" : "input", + "key" : 3.0, + "value" : { + "id" : 1 + } + }, { + "topic" : "input", + "key" : 2.0, + "value" : { + "id" : 2 + } + }, { + "topic" : "input", + "key" : 1.0, + "value" : { + "id" : 3 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3.0, + "value" : { + "ID" : 1, + "KEY" : 3.0 + } + }, { + "topic" : "OUTPUT", + "key" : 2.0, + "value" : { + "ID" : 2, + "KEY" : 2.0 + } + }, { + "topic" : "OUTPUT", + "key" : 1.0, + "value" : { + "ID" : 3, + "KEY" : 1.0 + } + } ], + "topics" : [ { + "name" : "input", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K DOUBLE PRIMARY KEY, ID bigint) WITH (kafka_topic='input',value_format='JSON');", "CREATE TABLE OUTPUT as SELECT K, ID, AS_VALUE(K) as KEY FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "K DOUBLE KEY, ID BIGINT, KEY DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + } + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "input", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/topology new file mode 100644 index 000000000000..f15bde0bc817 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_DOUBLE_KEY/6.1.0_1594164294154/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/plan.json new file mode 100644 index 000000000000..073a12f03ae0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/plan.json @@ -0,0 +1,130 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K INTEGER PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='input', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.ID ID,\n AS_VALUE(INPUT.K) KEY\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `ID` BIGINT, `KEY` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "AS_VALUE(K) AS KEY" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/spec.json new file mode 100644 index 000000000000..cfd410d0d99a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/spec.json @@ -0,0 +1,108 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164293990, + "path" : "query-validation-tests/key-schemas.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table explicit KAFKA INT KEY", + "inputs" : [ { + "topic" : "input", + "key" : 3, + "value" : { + "id" : 1 + } + }, { + "topic" : "input", + "key" : 2, + "value" : { + "id" : 2 + } + }, { + "topic" : "input", + "key" : 1, + "value" : { + "id" : 3 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "ID" : 1, + "KEY" : 3 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "ID" : 2, + "KEY" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 3, + "KEY" : 1 + } + } ], + "topics" : [ { + "name" : "input", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K INT PRIMARY KEY, ID bigint) WITH (kafka_topic='input',value_format='JSON');", "CREATE TABLE OUTPUT as SELECT K, ID, AS_VALUE(K) as KEY FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "K INT KEY, ID BIGINT, KEY INT", + "keyFormat" : { + "format" : "KAFKA" + } + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "input", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/topology new file mode 100644 index 000000000000..f15bde0bc817 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_INT_KEY/6.1.0_1594164293990/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/plan.json new file mode 100644 index 000000000000..2497ef917c85 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/plan.json @@ -0,0 +1,130 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='input', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.ID ID,\n AS_VALUE(INPUT.K) KEY\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT, `KEY` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "AS_VALUE(K) AS KEY" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/spec.json new file mode 100644 index 000000000000..fa4fa786ec28 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/spec.json @@ -0,0 +1,108 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164293900, + "path" : "query-validation-tests/key-schemas.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table explicit KAFKA STRING KEY", + "inputs" : [ { + "topic" : "input", + "key" : "1", + "value" : { + "id" : 1 + } + }, { + "topic" : "input", + "key" : "1", + "value" : { + "id" : 2 + } + }, { + "topic" : "input", + "key" : "", + "value" : { + "id" : 3 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "ID" : 1, + "KEY" : "1" + } + }, { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "ID" : 2, + "KEY" : "1" + } + }, { + "topic" : "OUTPUT", + "key" : "", + "value" : { + "ID" : 3, + "KEY" : "" + } + } ], + "topics" : [ { + "name" : "input", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID bigint) WITH (kafka_topic='input',value_format='JSON');", "CREATE TABLE OUTPUT as SELECT K, ID, AS_VALUE(K) as KEY FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "K STRING KEY, ID BIGINT, KEY STRING", + "keyFormat" : { + "format" : "KAFKA" + } + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "input", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/topology new file mode 100644 index 000000000000..f15bde0bc817 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_explicit_KAFKA_STRING_KEY/6.1.0_1594164293900/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/plan.json new file mode 100644 index 000000000000..2497ef917c85 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/plan.json @@ -0,0 +1,130 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='input', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.ID ID,\n AS_VALUE(INPUT.K) KEY\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT, `KEY` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "AS_VALUE(K) AS KEY" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/spec.json new file mode 100644 index 000000000000..c8e372457ff1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/spec.json @@ -0,0 +1,108 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164293814, + "path" : "query-validation-tests/key-schemas.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table implicit KAFKA STRING KEY", + "inputs" : [ { + "topic" : "input", + "key" : "1", + "value" : { + "id" : 1 + } + }, { + "topic" : "input", + "key" : "1", + "value" : { + "id" : 2 + } + }, { + "topic" : "input", + "key" : "", + "value" : { + "id" : 3 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "ID" : 1, + "KEY" : "1" + } + }, { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "ID" : 2, + "KEY" : "1" + } + }, { + "topic" : "OUTPUT", + "key" : "", + "value" : { + "ID" : 3, + "KEY" : "" + } + } ], + "topics" : [ { + "name" : "input", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID bigint) WITH (kafka_topic='input',value_format='JSON');", "CREATE TABLE OUTPUT as SELECT K, ID, AS_VALUE(K) as KEY FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "K STRING KEY, ID BIGINT, KEY STRING", + "keyFormat" : { + "format" : "KAFKA" + } + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "input", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/topology new file mode 100644 index 000000000000..f15bde0bc817 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/key-schemas_-_table_implicit_KAFKA_STRING_KEY/6.1.0_1594164293814/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/plan.json new file mode 100644 index 000000000000..eeddefcd73eb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/spec.json new file mode 100644 index 000000000000..d2f4cd73bada --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/spec.json @@ -0,0 +1,140 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164296549, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "first join column in projection", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_first_join_column_in_projection/6.1.0_1594164296549/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/plan.json new file mode 100644 index 000000000000..58fd1f3f2a40 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T2.ID T2_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T2_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T2_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "T2_ID" + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/spec.json new file mode 100644 index 000000000000..70bba9a72d12 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/spec.json @@ -0,0 +1,140 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164296667, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "second join column in projection", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT T2.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "T2_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_second_join_column_in_projection/6.1.0_1594164296667/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/plan.json new file mode 100644 index 000000000000..c72ab980164a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, DIFF BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `DIFF` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.DIFF DIFF\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `DIFF` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DIFF` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "DIFF AS T3_DIFF", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_DIFF AS DIFF" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/spec.json new file mode 100644 index 000000000000..7678788c9a53 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/spec.json @@ -0,0 +1,148 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164300902, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "should generate correct aliases when sources have some matching and some different columns", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "diff" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "V0" : 1 + }, + "timestamp" : 14 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "DIFF" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, DIFF bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.ID, s1.V0, t2.V0, t3.DIFF FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, DIFF BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_should_generate_correct_aliases_when_sources_have_some_matching_and_some_different_columns/6.1.0_1594164300902/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/plan.json new file mode 100644 index 000000000000..954b3f969097 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/plan.json @@ -0,0 +1,271 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nFULL OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "OUTER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "ROWKEY" + }, + "keyExpression" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/spec.json new file mode 100644 index 000000000000..ac7fc0c6b032 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/spec.json @@ -0,0 +1,279 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164299923, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-stream-table - full-inner", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : null, + "value" : { + "S1_V0" : null, + "S1_ROWTIME" : null, + "S1_ID" : null, + "S2_V0" : 2, + "S2_ROWTIME" : 11, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 11 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S1_ROWTIME" : 12, + "S1_ID" : 0, + "S2_V0" : 2, + "S2_ROWTIME" : 11, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 12 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S1_ROWTIME" : 12, + "S1_ID" : 0, + "S2_V0" : 4, + "S2_ROWTIME" : 13, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 13 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S1_ROWTIME" : 100000, + "S1_ID" : 0, + "S2_V0" : null, + "S2_ROWTIME" : null, + "S2_ID" : null, + "KSQL_COL_0" : null + }, + "timestamp" : 100000 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S1_ROWTIME" : 100000, + "S1_ID" : 0, + "S2_V0" : 6, + "S2_ROWTIME" : 100001, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 100001 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 4, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : null, + "T3_V0" : 3 + }, + "timestamp" : 100000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 6, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.ID, s1.V0, s2.V0, t3.V0 FROM S1 FULL JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, S2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000015-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTERTHIS-0000000014-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/topology new file mode 100644 index 000000000000..72b83e356b67 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner/6.1.0_1594164299923/topology @@ -0,0 +1,68 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000023 + <-- Join + Sink: KSTREAM-SINK-0000000023 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000009 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000010 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000006 + Processor: KSTREAM-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-SOURCE-0000000009 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000010 + Processor: L_Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000015-store]) + --> L_Join-outer-other-join + <-- PrependAliasL_Right + Processor: L_Join-this-windowed (stores: [KSTREAM-OUTERTHIS-0000000014-store]) + --> L_Join-outer-this-join + <-- PrependAliasL_Left + Processor: L_Join-outer-other-join (stores: [KSTREAM-OUTERTHIS-0000000014-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-outer-this-join (stores: [KSTREAM-OUTEROTHER-0000000015-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: L_Join-merge (stores: []) + --> LeftSourceKeyed-SelectKey + <-- L_Join-outer-this-join, L_Join-outer-other-join + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> Join-repartition-filter + <-- L_Join-merge + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- LeftSourceKeyed-SelectKey + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/plan.json new file mode 100644 index 000000000000..74c42e4de55d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/plan.json @@ -0,0 +1,271 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM S1 S1\nFULL OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_ID` INTEGER, `S2_V0` BIGINT, `T3_ID` INTEGER, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "OUTER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "ROWKEY" + }, + "keyExpression" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_ID AS S2_ID", "S2_V0 AS S2_V0", "T3_ID AS T3_ID", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/spec.json new file mode 100644 index 000000000000..851a455b5099 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/spec.json @@ -0,0 +1,287 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164300054, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-stream-table - full-inner - select *", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : null, + "value" : { + "S1_V0" : null, + "S1_ROWTIME" : null, + "S1_ID" : null, + "S2_V0" : 2, + "S2_ROWTIME" : 11, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 11 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S1_ROWTIME" : 12, + "S1_ID" : 0, + "S2_V0" : 2, + "S2_ROWTIME" : 11, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 12 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S1_ROWTIME" : 12, + "S1_ID" : 0, + "S2_V0" : 4, + "S2_ROWTIME" : 13, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 13 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S1_ROWTIME" : 100000, + "S1_ID" : 0, + "S2_V0" : null, + "S2_ROWTIME" : null, + "S2_ID" : null, + "KSQL_COL_0" : null + }, + "timestamp" : 100000 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S1_ROWTIME" : 100000, + "S1_ID" : 0, + "S2_V0" : 6, + "S2_ROWTIME" : 100001, + "S2_ID" : 0, + "KSQL_COL_0" : null + }, + "timestamp" : 100001 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_ID" : 0, + "S2_V0" : 2, + "T3_ID" : 0, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_ID" : 0, + "S2_V0" : 4, + "T3_ID" : 0, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_ID" : null, + "S2_V0" : null, + "T3_ID" : 0, + "T3_V0" : 3 + }, + "timestamp" : 100000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_ID" : 0, + "S2_V0" : 6, + "T3_ID" : 0, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT * FROM S1 FULL JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, S2_ID INT, S2_V0 BIGINT, T3_ID INT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000015-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTERTHIS-0000000014-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/topology new file mode 100644 index 000000000000..72b83e356b67 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_full-inner_-_select__/6.1.0_1594164300054/topology @@ -0,0 +1,68 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000023 + <-- Join + Sink: KSTREAM-SINK-0000000023 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000009 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000010 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000006 + Processor: KSTREAM-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-SOURCE-0000000009 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000010 + Processor: L_Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000015-store]) + --> L_Join-outer-other-join + <-- PrependAliasL_Right + Processor: L_Join-this-windowed (stores: [KSTREAM-OUTERTHIS-0000000014-store]) + --> L_Join-outer-this-join + <-- PrependAliasL_Left + Processor: L_Join-outer-other-join (stores: [KSTREAM-OUTERTHIS-0000000014-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-outer-this-join (stores: [KSTREAM-OUTEROTHER-0000000015-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: L_Join-merge (stores: []) + --> LeftSourceKeyed-SelectKey + <-- L_Join-outer-this-join, L_Join-outer-other-join + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> Join-repartition-filter + <-- L_Join-merge + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- LeftSourceKeyed-SelectKey + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/plan.json new file mode 100644 index 000000000000..e01247ef87f6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/plan.json @@ -0,0 +1,264 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S2.ID S2_ID,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S2_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S2_ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/spec.json new file mode 100644 index 000000000000..cde7bb3569c1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/spec.json @@ -0,0 +1,191 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164299126, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-stream-table - inner-inner", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 4, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 6, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S2_ID INT KEY, S1_V0 BIGINT, S2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000015-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000014-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/topology new file mode 100644 index 000000000000..fc856d3bd0f2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner/6.1.0_1594164299126/topology @@ -0,0 +1,56 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000009 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000010 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000006 + Processor: KSTREAM-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-SOURCE-0000000009 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000010 + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000015-store]) + --> L_Join-other-join + <-- PrependAliasL_Right + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000014-store]) + --> L_Join-this-join + <-- PrependAliasL_Left + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000014-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000015-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: L_Join-merge (stores: []) + --> Join + <-- L_Join-this-join, L_Join-other-join + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join-merge + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000019 + <-- Join + Sink: KSTREAM-SINK-0000000019 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/plan.json new file mode 100644 index 000000000000..0d80db699414 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/plan.json @@ -0,0 +1,278 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.K S1_K,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.K = S2.K))\nINNER JOIN T3 T3 ON ((S2.K = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_K` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "K" + }, + "keyColumnNames" : [ "S1_K" ], + "selectExpressions" : [ "K AS S1_K", "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "K" + }, + "keyColumnNames" : [ "S2_K" ], + "selectExpressions" : [ "K AS S2_K", "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_K" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_K" + }, + "keyColumnNames" : [ "S1_K" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/spec.json new file mode 100644 index 000000000000..37fb607992c1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/spec.json @@ -0,0 +1,215 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164299646, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-stream-table - inner-inner - cascading join criteria", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 2, + "value" : { + "k" : 0, + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "k" : 0, + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 2, + "value" : { + "k" : 0, + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "k" : 0, + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 2, + "value" : { + "k" : 0, + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 4, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 6, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, k int, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, k int, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT s1.k, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.K = S2.K JOIN T3 ON S2.K = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_K INT KEY, S1_V0 BIGINT, S2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000023-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000022-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/topology new file mode 100644 index 000000000000..b432cf924eb2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_cascading_join_criteria/6.1.0_1594164299646/topology @@ -0,0 +1,80 @@ +Topologies: + Sub-topology: 0 + Source: L_Join-left-repartition-source (topics: [L_Join-left-repartition]) + --> L_Join-this-windowed + Source: L_Join-right-repartition-source (topics: [L_Join-right-repartition]) + --> L_Join-other-windowed + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000023-store]) + --> L_Join-other-join + <-- L_Join-right-repartition-source + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000022-store]) + --> L_Join-this-join + <-- L_Join-left-repartition-source + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000022-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000023-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: L_Join-merge (stores: []) + --> Join + <-- L_Join-this-join, L_Join-other-join + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join-merge + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000027 + <-- Join + Sink: KSTREAM-SINK-0000000027 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000006 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-left-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-left-repartition-filter (stores: []) + --> L_Join-left-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-left-repartition-sink (topic: L_Join-left-repartition) + <-- L_Join-left-repartition-filter + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000010 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000011 + Processor: KSTREAM-TRANSFORMVALUES-0000000011 (stores: []) + --> L_RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000010 + Processor: L_RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-TRANSFORMVALUES-0000000011 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-right-repartition-filter + <-- L_RightSourceKeyed-SelectKey + Processor: L_Join-right-repartition-filter (stores: []) + --> L_Join-right-repartition-sink + <-- PrependAliasL_Right + Sink: L_Join-right-repartition-sink (topic: L_Join-right-repartition) + <-- L_Join-right-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/plan.json new file mode 100644 index 000000000000..e6b9e4e2d01c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/plan.json @@ -0,0 +1,271 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.K K,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.K = S2.ID))\nINNER JOIN T3 T3 ON ((S1.K = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "K" + }, + "keyColumnNames" : [ "S1_K" ], + "selectExpressions" : [ "K AS S1_K", "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_K" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_K" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/spec.json new file mode 100644 index 000000000000..09368681d917 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/spec.json @@ -0,0 +1,202 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164299254, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-stream-table - inner-inner - rekey left", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "k" : 0, + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "k" : 0, + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 4, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 6, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, k int, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.K, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.K = S2.ID JOIN T3 ON S1.K = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "K INT KEY, S1_V0 BIGINT, S2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000019-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000018-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/topology new file mode 100644 index 000000000000..2070bd6ba86c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left/6.1.0_1594164299254/topology @@ -0,0 +1,68 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000010 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000011 + Processor: KSTREAM-TRANSFORMVALUES-0000000011 (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-SOURCE-0000000010 + Source: L_Join-left-repartition-source (topics: [L_Join-left-repartition]) + --> L_Join-this-windowed + Processor: PrependAliasL_Right (stores: []) + --> L_Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000011 + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000019-store]) + --> L_Join-other-join + <-- PrependAliasL_Right + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000018-store]) + --> L_Join-this-join + <-- L_Join-left-repartition-source + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000018-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000019-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: L_Join-merge (stores: []) + --> Join + <-- L_Join-this-join, L_Join-other-join + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join-merge + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000023 + <-- Join + Sink: KSTREAM-SINK-0000000023 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000006 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-left-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-left-repartition-filter (stores: []) + --> L_Join-left-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-left-repartition-sink (topic: L_Join-left-repartition) + <-- L_Join-left-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/plan.json new file mode 100644 index 000000000000..3d4ce44f9ff0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/plan.json @@ -0,0 +1,278 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T3.ID T3_ID,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.K = S2.K))\nINNER JOIN T3 T3 ON ((S1.K = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T3_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "K" + }, + "keyColumnNames" : [ "S1_K" ], + "selectExpressions" : [ "K AS S1_K", "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "K" + }, + "keyColumnNames" : [ "S2_K" ], + "selectExpressions" : [ "K AS S2_K", "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_K" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "T3_ID" + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/spec.json new file mode 100644 index 000000000000..9f86f8217e13 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/spec.json @@ -0,0 +1,275 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164299387, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-stream-table - inner-inner - rekey left and right", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 2, + "value" : { + "k" : 0, + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "k" : 0, + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 2, + "value" : { + "k" : 0, + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "k" : 0, + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 2, + "value" : { + "k" : 0, + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "key" : 0, + "value" : { + "S2_K" : 0, + "S2_V0" : 2, + "S2_ROWTIME" : 11, + "S2_ID" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "key" : 0, + "value" : { + "S2_K" : 0, + "S2_V0" : 4, + "S2_ROWTIME" : 13, + "S2_ID" : 2 + }, + "timestamp" : 13 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "key" : 0, + "value" : { + "S2_K" : 0, + "S2_V0" : 6, + "S2_ROWTIME" : 100001, + "S2_ID" : 2 + }, + "timestamp" : 100001 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "key" : 0, + "value" : { + "S1_K" : 0, + "S1_V0" : 1, + "S1_ROWTIME" : 12, + "S1_ID" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "key" : 0, + "value" : { + "S1_K" : 0, + "S1_V0" : 5, + "S1_ROWTIME" : 100000, + "S1_ID" : 1 + }, + "timestamp" : 100000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 4, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 6, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, k int, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, k int, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT T3.ID, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.K = S2.K JOIN T3 ON S1.K = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "T3_ID INT KEY, S1_V0 BIGINT, S2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000023-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000022-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/topology new file mode 100644 index 000000000000..b432cf924eb2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_inner-inner_-_rekey_left_and_right/6.1.0_1594164299387/topology @@ -0,0 +1,80 @@ +Topologies: + Sub-topology: 0 + Source: L_Join-left-repartition-source (topics: [L_Join-left-repartition]) + --> L_Join-this-windowed + Source: L_Join-right-repartition-source (topics: [L_Join-right-repartition]) + --> L_Join-other-windowed + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000023-store]) + --> L_Join-other-join + <-- L_Join-right-repartition-source + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000022-store]) + --> L_Join-this-join + <-- L_Join-left-repartition-source + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000022-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000023-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: L_Join-merge (stores: []) + --> Join + <-- L_Join-this-join, L_Join-other-join + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join-merge + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000027 + <-- Join + Sink: KSTREAM-SINK-0000000027 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000006 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-left-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-left-repartition-filter (stores: []) + --> L_Join-left-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-left-repartition-sink (topic: L_Join-left-repartition) + <-- L_Join-left-repartition-filter + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000010 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000011 + Processor: KSTREAM-TRANSFORMVALUES-0000000011 (stores: []) + --> L_RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000010 + Processor: L_RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-TRANSFORMVALUES-0000000011 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-right-repartition-filter + <-- L_RightSourceKeyed-SelectKey + Processor: L_Join-right-repartition-filter (stores: []) + --> L_Join-right-repartition-sink + <-- PrependAliasL_Right + Sink: L_Join-right-repartition-sink (topic: L_Join-right-repartition) + <-- L_Join-right-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/plan.json new file mode 100644 index 000000000000..850331a8159b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/plan.json @@ -0,0 +1,264 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0,\n S2.ID S2_ID\nFROM S1 S1\nLEFT OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S2_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S2_ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/spec.json new file mode 100644 index 000000000000..72763bc69b42 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/spec.json @@ -0,0 +1,200 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164299805, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-stream-table - left-inner", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 4, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : null, + "T3_V0" : 3 + }, + "timestamp" : 100000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 6, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT s1.V0, s2.V0, t3.V0, S2.ID FROM S1 LEFT JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S2_ID INT KEY, S1_V0 BIGINT, S2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000014-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000015-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/topology new file mode 100644 index 000000000000..6a751df2576c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_left-inner/6.1.0_1594164299805/topology @@ -0,0 +1,56 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000009 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000010 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000006 + Processor: KSTREAM-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-SOURCE-0000000009 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000010 + Processor: L_Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000015-store]) + --> L_Join-outer-other-join + <-- PrependAliasL_Right + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000014-store]) + --> L_Join-this-join + <-- PrependAliasL_Left + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Processor: L_Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000014-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000015-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: L_Join-merge (stores: []) + --> Join + <-- L_Join-this-join, L_Join-outer-other-join + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join-merge + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000019 + <-- Join + Sink: KSTREAM-SINK-0000000019 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/plan.json new file mode 100644 index 000000000000..9b80d5e77e2d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/plan.json @@ -0,0 +1,264 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S3 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S3.ID S3_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n S3.V0 S3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN S3 S3 WITHIN 10 SECONDS ON ((S1.ID = S3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S3_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `S3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S3", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S3_ID" ], + "selectExpressions" : [ "V0 AS S3_V0", "ROWTIME AS S3_ROWTIME", "ID AS S3_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S3_ID" + }, + "keyColumnNames" : [ "S3_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "S3_V0 AS S3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/spec.json new file mode 100644 index 000000000000..28dd1080bab6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/spec.json @@ -0,0 +1,182 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164300181, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-stream - inner-inner", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "S3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "T2_V0" : 4, + "S3_V0" : 6 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE STREAM S3 (ID INT KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S3.ID, s1.V0, t2.V0, s3.V0 FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN S3 WITHIN 10 seconds ON S1.ID = S3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S3_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, S3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000016-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000015-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/topology new file mode 100644 index 000000000000..4df0c9f02665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_inner-inner/6.1.0_1594164300181/topology @@ -0,0 +1,56 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000010 (topics: [right2]) + --> KSTREAM-TRANSFORMVALUES-0000000011 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000006 + Processor: KSTREAM-TRANSFORMVALUES-0000000011 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000010 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join-this-windowed + <-- PrependAliasL_Left + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000011 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000016-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000015-store]) + --> Join-this-join + <-- L_Join + Source: KSTREAM-SOURCE-0000000001 (topics: [right]) + --> KTABLE-SOURCE-0000000002 + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000015-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000016-store]) + --> Join-merge + <-- Join-this-windowed + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000019 + <-- Join-merge + Sink: KSTREAM-SINK-0000000019 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/plan.json new file mode 100644 index 000000000000..d3f4ad75e226 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/plan.json @@ -0,0 +1,264 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S3 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S3.ID S3_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n S3.V0 S3_V0\nFROM S1 S1\nLEFT OUTER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN S3 S3 WITHIN 10 SECONDS ON ((S1.ID = S3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S3_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `S3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S3", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S3_ID" ], + "selectExpressions" : [ "V0 AS S3_V0", "ROWTIME AS S3_ROWTIME", "ID AS S3_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S3_ID" + }, + "keyColumnNames" : [ "S3_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "S3_V0 AS S3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/spec.json new file mode 100644 index 000000000000..e309b2564cd7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/spec.json @@ -0,0 +1,182 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164300302, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-stream - left-inner", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : null, + "S3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "T2_V0" : 4, + "S3_V0" : 6 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE STREAM S3 (ID INT KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S3.ID, s1.V0, t2.V0, s3.V0 FROM S1 LEFT JOIN T2 ON S1.ID = T2.ID JOIN S3 WITHIN 10 seconds ON S1.ID = S3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S3_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, S3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000016-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000015-store-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/topology new file mode 100644 index 000000000000..4df0c9f02665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-stream_-_left-inner/6.1.0_1594164300302/topology @@ -0,0 +1,56 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000010 (topics: [right2]) + --> KSTREAM-TRANSFORMVALUES-0000000011 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000006 + Processor: KSTREAM-TRANSFORMVALUES-0000000011 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000010 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join-this-windowed + <-- PrependAliasL_Left + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000011 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000016-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000015-store]) + --> Join-this-join + <-- L_Join + Source: KSTREAM-SOURCE-0000000001 (topics: [right]) + --> KTABLE-SOURCE-0000000002 + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000015-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000016-store]) + --> Join-merge + <-- Join-this-windowed + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000019 + <-- Join-merge + Sink: KSTREAM-SINK-0000000019 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/plan.json new file mode 100644 index 000000000000..3ecddf99bff1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n *,\n S1.ROWTIME S1_ROWTIME,\n T2.ROWTIME T2_ROWTIME,\n T3.ROWTIME T3_ROWTIME\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_ID` INTEGER, `T2_V0` BIGINT, `T3_ID` INTEGER, `T3_V0` BIGINT, `S1_ROWTIME` BIGINT, `T2_ROWTIME` BIGINT, `T3_ROWTIME` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_ID AS T2_ID", "T2_V0 AS T2_V0", "T3_ID AS T3_ID", "T3_V0 AS T3_V0", "S1_ROWTIME AS S1_ROWTIME", "T2_ROWTIME AS T2_ROWTIME", "T3_ROWTIME AS T3_ROWTIME" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/spec.json new file mode 100644 index 000000000000..43e178002d2e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/spec.json @@ -0,0 +1,146 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164298243, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - explicit ROWTIME", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_ID" : 0, + "T2_V0" : 2, + "T3_ID" : 0, + "T3_V0" : 3, + "S1_ROWTIME" : 12, + "T2_ROWTIME" : 12, + "T3_ROWTIME" : 12 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT *, S1.ROWTIME, T2.ROWTIME, T3.ROWTIME FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_ID INT, T2_V0 BIGINT, T3_ID INT, T3_V0 BIGINT, S1_ROWTIME BIGINT, T2_ROWTIME BIGINT, T3_ROWTIME BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_explicit_ROWTIME/6.1.0_1594164298243/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/plan.json new file mode 100644 index 000000000000..eeddefcd73eb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/spec.json new file mode 100644 index 000000000000..05d9b270360e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/spec.json @@ -0,0 +1,148 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164297093, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "V0" : 1 + }, + "timestamp" : 14 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner/6.1.0_1594164297093/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/plan.json new file mode 100644 index 000000000000..3f6884ada134 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S.ID S_ID,\n S.V0 S_V0,\n T.V0 T_V0,\n TT.V0 TT_V0\nFROM S1 S\nINNER JOIN T2 T ON ((S.ID = T.ID))\nINNER JOIN T3 TT ON ((S.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S_ID` INTEGER KEY, `S_V0` BIGINT, `T_V0` BIGINT, `TT_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "V0 AS S_V0", "ROWTIME AS S_ROWTIME", "ID AS S_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "V0 AS T_V0", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "keyColName" : "S_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "V0 AS TT_V0", "ROWTIME AS TT_ROWTIME", "ID AS TT_ID" ] + }, + "keyColName" : "S_ID" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "S_V0 AS S_V0", "T_V0 AS T_V0", "TT_V0 AS TT_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/spec.json new file mode 100644 index 000000000000..b0ce883abe03 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/spec.json @@ -0,0 +1,148 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164299012, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - aliasing", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "V0" : 1 + }, + "timestamp" : 14 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S_V0" : 1, + "T_V0" : 2, + "TT_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S.ID, s.V0, t.V0, tt.V0 FROM S1 s JOIN T2 t ON S.ID = T.ID JOIN T3 tt ON S.ID = TT.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S_ID INT KEY, S_V0 BIGINT, T_V0 BIGINT, TT_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_aliasing/6.1.0_1594164299012/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/plan.json new file mode 100644 index 000000000000..4eb960dc1838 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T2.ID T2_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((T2.ID = S1.ID))\nINNER JOIN T3 T3 ON ((T3.ID = S1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T2_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T2_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "T2_ID" + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/spec.json new file mode 100644 index 000000000000..33bf11ce3d2b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/spec.json @@ -0,0 +1,148 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164298814, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - flip join expression", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "left", + "key" : 1, + "value" : { + "V0" : 1 + }, + "timestamp" : 14 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT T2.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON T2.ID = S1.ID JOIN T3 ON T3.ID = S1.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "T2_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_flip_join_expression/6.1.0_1594164298814/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/plan.json new file mode 100644 index 000000000000..e40082bfea54 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/plan.json @@ -0,0 +1,262 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.K K,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.K = T2.ID))\nINNER JOIN T3 T3 ON ((S1.K = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "K" + }, + "keyColumnNames" : [ "S1_K" ], + "selectExpressions" : [ "K AS S1_K", "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_K" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_K" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/spec.json new file mode 100644 index 000000000000..65345674d204 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/spec.json @@ -0,0 +1,166 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164297706, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - rekey", + "inputs" : [ { + "topic" : "right2", + "key" : 1, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 1, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "k" : 1, + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "key" : 1, + "value" : { + "S1_K" : 1, + "S1_V0" : 1, + "S1_ROWTIME" : 12, + "S1_ID" : 0 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, K INT, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.K, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.K = T2.ID JOIN T3 ON S1.K = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "K INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/topology new file mode 100644 index 000000000000..aff90b07d0d2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey/6.1.0_1594164297706/topology @@ -0,0 +1,62 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Source: L_Join-repartition-source (topics: [L_Join-repartition]) + --> L_Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- L_Join-repartition-source + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000022 + <-- Join + Sink: KSTREAM-SINK-0000000022 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-repartition-filter (stores: []) + --> L_Join-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-repartition-sink (topic: L_Join-repartition) + <-- L_Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/plan.json new file mode 100644 index 000000000000..0998dc093b89 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/plan.json @@ -0,0 +1,262 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T3.ID T3_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((T2.ID = S1.K))\nINNER JOIN T3 T3 ON ((T3.ID = S1.K))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T3_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "K" + }, + "keyColumnNames" : [ "S1_K" ], + "selectExpressions" : [ "K AS S1_K", "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_K" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "T3_ID" + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/spec.json new file mode 100644 index 000000000000..6ca0e4e98db9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/spec.json @@ -0,0 +1,166 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164298916, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - rekey and flip join expression", + "inputs" : [ { + "topic" : "right2", + "key" : 1, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 1, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "k" : 1, + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "key" : 1, + "value" : { + "S1_K" : 1, + "S1_V0" : 1, + "S1_ROWTIME" : 12, + "S1_ID" : 0 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, K INT, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT T3.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON T2.ID = S1.K JOIN T3 ON T3.ID = S1.K;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "T3_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/topology new file mode 100644 index 000000000000..aff90b07d0d2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_and_flip_join_expression/6.1.0_1594164298916/topology @@ -0,0 +1,62 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Source: L_Join-repartition-source (topics: [L_Join-repartition]) + --> L_Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- L_Join-repartition-source + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000022 + <-- Join + Sink: KSTREAM-SINK-0000000022 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-repartition-filter (stores: []) + --> L_Join-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-repartition-sink (topic: L_Join-repartition) + <-- L_Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/plan.json new file mode 100644 index 000000000000..eee8b59681a9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/plan.json @@ -0,0 +1,269 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, K INTEGER, V0 INTEGER) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` INTEGER", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 INTEGER) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 INTEGER) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.K K,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.V0 = T2.ID))\nINNER JOIN T3 T3 ON ((S1.K = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `S1_V0` INTEGER, `T2_V0` INTEGER, `T3_V0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` INTEGER" + }, + "keyExpression" : "V0" + }, + "keyColumnNames" : [ "S1_V0" ], + "selectExpressions" : [ "K AS S1_K", "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_V0" + }, + "keyExpression" : "S1_K" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_K" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/spec.json new file mode 100644 index 000000000000..f2853ecc6fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/spec.json @@ -0,0 +1,161 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164297985, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - rekey on different expression", + "inputs" : [ { + "topic" : "right2", + "key" : 2, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 1, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "k" : 2, + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, K INT, V0 int) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 int) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 int) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.K, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.V0 = T2.ID JOIN T3 ON S1.K = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "K INT KEY, S1_V0 INT, T2_V0 INT, T3_V0 INT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/topology new file mode 100644 index 000000000000..3407adf8d604 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_on_different_expression/6.1.0_1594164297985/topology @@ -0,0 +1,74 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000026 + <-- Join + Sink: KSTREAM-SINK-0000000026 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Source: L_Join-repartition-source (topics: [L_Join-repartition]) + --> L_Join + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> LeftSourceKeyed-SelectKey + <-- L_Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> Join-repartition-filter + <-- L_Join + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- LeftSourceKeyed-SelectKey + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-repartition-filter (stores: []) + --> L_Join-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-repartition-sink (topic: L_Join-repartition) + <-- L_Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/plan.json new file mode 100644 index 000000000000..59710961743a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/plan.json @@ -0,0 +1,269 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, K INTEGER, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T3.ID T3_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON (((S1.K - 1) = T2.ID))\nINNER JOIN T3 T3 ON (((S1.K + 1) = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T3_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `K` INTEGER, `V0` BIGINT" + }, + "keyExpression" : "(K - 1)" + }, + "keyColumnNames" : [ "S1_KSQL_COL_0" ], + "selectExpressions" : [ "K AS S1_K", "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID", "KSQL_COL_0 AS S1_KSQL_COL_0" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T2_ID" + }, + "keyExpression" : "(S1_K + 1)" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "T3_ID" + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/spec.json new file mode 100644 index 000000000000..048ea1dbe7cf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/spec.json @@ -0,0 +1,161 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164297838, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - rekey with different expression", + "inputs" : [ { + "topic" : "right2", + "key" : 3, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 1, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "k" : 2, + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, K INT, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT T3.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.K - 1 = T2.ID JOIN T3 ON S1.K + 1 = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "T3_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-repartition", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/topology new file mode 100644 index 000000000000..3407adf8d604 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_rekey_with_different_expression/6.1.0_1594164297838/topology @@ -0,0 +1,74 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000026 + <-- Join + Sink: KSTREAM-SINK-0000000026 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Source: L_Join-repartition-source (topics: [L_Join-repartition]) + --> L_Join + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> LeftSourceKeyed-SelectKey + <-- L_Join-repartition-source + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> Join-repartition-filter + <-- L_Join + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- LeftSourceKeyed-SelectKey + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-repartition-filter (stores: []) + --> L_Join-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-repartition-sink (topic: L_Join-repartition) + <-- L_Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/plan.json new file mode 100644 index 000000000000..90147c60dadd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_ID` INTEGER, `T2_V0` BIGINT, `T3_ID` INTEGER, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_ID AS T2_ID", "T2_V0 AS T2_V0", "T3_ID AS T3_ID", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/spec.json new file mode 100644 index 000000000000..7a59b3c79ab6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/spec.json @@ -0,0 +1,143 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164298399, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - select * ", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_ID" : 0, + "T2_V0" : 2, + "T3_ID" : 0, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT * FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_ID INT, T2_V0 BIGINT, T3_ID INT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select___/6.1.0_1594164298399/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/plan.json new file mode 100644 index 000000000000..808d5a4568e8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.*,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/spec.json new file mode 100644 index 000000000000..00180c15c9ac --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/spec.json @@ -0,0 +1,141 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164298620, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - select left.* and specific fields from rights", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT s1.*, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__and_specific_fields_from_rights/6.1.0_1594164298620/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/plan.json new file mode 100644 index 000000000000..153e20fb8105 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT S1.*\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/spec.json new file mode 100644 index 000000000000..6f7a20752cd4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/spec.json @@ -0,0 +1,139 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164298520, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - select left.* only", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT s1.* FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_left.__only/6.1.0_1594164298520/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/plan.json new file mode 100644 index 000000000000..e1eb79946067 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.*,\n T2.*,\n T3.*\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_ID` INTEGER, `T2_V0` BIGINT, `T3_ID` INTEGER, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_ID AS T2_ID", "T2_V0 AS T2_V0", "T3_ID AS T3_ID", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/spec.json new file mode 100644 index 000000000000..5daf165afd08 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/spec.json @@ -0,0 +1,143 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164298715, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-inner - select sources.*", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_ID" : 0, + "T2_V0" : 2, + "T3_ID" : 0, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT s1.*, t2.*, t3.* FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_ID INT, T2_V0 BIGINT, T3_ID INT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-inner_-_select_sources._/6.1.0_1594164298715/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/plan.json new file mode 100644 index 000000000000..b99238b69a31 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nLEFT OUTER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/spec.json new file mode 100644 index 000000000000..48fa6dd9cbfb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/spec.json @@ -0,0 +1,164 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164297464, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - inner-left", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 9 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 10 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 11 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 13 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : null + }, + "timestamp" : 11 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 13 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.ID = T2.ID LEFT JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_inner-left/6.1.0_1594164297464/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/plan.json new file mode 100644 index 000000000000..c5a6879ee581 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nLEFT OUTER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/spec.json new file mode 100644 index 000000000000..f4ecefe89860 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/spec.json @@ -0,0 +1,155 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164297333, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - left-inner", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 9 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 10 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 11 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 13 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 13 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.ID, s1.V0, t2.V0, t3.V0 FROM S1 LEFT JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-inner/6.1.0_1594164297333/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/plan.json new file mode 100644 index 000000000000..8811b5c4d8af --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nLEFT OUTER JOIN T2 T2 ON ((S1.ID = T2.ID))\nLEFT OUTER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/spec.json new file mode 100644 index 000000000000..e7f41b9da2cf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/spec.json @@ -0,0 +1,173 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164297215, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "stream-table-table - left-left", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 9 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 10 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 11 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 13 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : null, + "T3_V0" : null + }, + "timestamp" : 9 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : null + }, + "timestamp" : 11 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 13 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S1.ID, s1.V0, t2.V0, t3.V0 FROM S1 LEFT JOIN T2 ON S1.ID = T2.ID LEFT JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "S1_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-table-table_-_left-left/6.1.0_1594164297215/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/plan.json new file mode 100644 index 000000000000..81a0aa394c5a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/plan.json @@ -0,0 +1,240 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n ROWKEY ID,\n T1.V0 T1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM T1 T1\nINNER JOIN T2 T2 ON ((T1.ID = T2.ID))\nFULL OUTER JOIN T3 T3 ON ((T1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` INTEGER KEY, `T1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "V0 AS T1_V0", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T1_V0 AS T1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/spec.json new file mode 100644 index 000000000000..5a545217b20c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164300694, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table-table-table - inner-full", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 3 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 1000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : null, + "T2_V0" : null, + "T3_V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 3 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : 4, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 1000 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT ROWKEY AS ID, T1.V0, T2.V0, T3.V0 FROM T1 JOIN T2 ON T1.ID = T2.ID FULL JOIN T3 ON T1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "ID INT PRIMARY KEY, T1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/topology new file mode 100644 index 000000000000..12fa3267efd8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-full/6.1.0_1594164300694/topology @@ -0,0 +1,71 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: [left-STATE-STORE-0000000000]) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: [right-STATE-STORE-0000000006]) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Source: KSTREAM-SOURCE-0000000016 (topics: [right2]) + --> KTABLE-SOURCE-0000000017 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000017 (stores: [right2-STATE-STORE-0000000015]) + --> KTABLE-MAPVALUES-0000000018 + <-- KSTREAM-SOURCE-0000000016 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-MAPVALUES-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000019 + <-- KTABLE-SOURCE-0000000017 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasL_Right + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasL_Left + Processor: KTABLE-TRANSFORMVALUES-0000000019 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000018 + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-JOINTHIS-0000000022 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000023 + <-- KTABLE-TRANSFORMVALUES-0000000019 + Processor: KTABLE-JOINOTHER-0000000023 (stores: [KafkaTopic_L_Left-Reduce, KafkaTopic_L_Right-Reduce]) + --> KTABLE-MERGE-0000000021 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000022 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000021 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-MERGE-0000000021 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000022, KTABLE-JOINOTHER-0000000023 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000025 + <-- KTABLE-MERGE-0000000021 + Processor: KTABLE-TOSTREAM-0000000025 (stores: []) + --> KSTREAM-SINK-0000000026 + <-- Project + Sink: KSTREAM-SINK-0000000026 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000025 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/plan.json new file mode 100644 index 000000000000..af1ba6201688 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/plan.json @@ -0,0 +1,240 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n T1.V0 T1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM T1 T1\nINNER JOIN T2 T2 ON ((T1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((T1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` INTEGER KEY, `T1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "V0 AS T1_V0", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_V0 AS T1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/spec.json new file mode 100644 index 000000000000..8820aebfd692 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/spec.json @@ -0,0 +1,165 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164300564, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "table-table-table - inner-inner", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 1 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 1000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : 4, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 1000 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT T1.ID, T1.V0, T2.V0, T3.V0 FROM T1 JOIN T2 ON T1.ID = T2.ID JOIN T3 ON T1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "table", + "schema" : "T1_ID INT PRIMARY KEY, T1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/topology new file mode 100644 index 000000000000..a23b1a903061 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_inner-inner/6.1.0_1594164300564/topology @@ -0,0 +1,71 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Source: KSTREAM-SOURCE-0000000016 (topics: [right2]) + --> KTABLE-SOURCE-0000000017 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000017 (stores: []) + --> KTABLE-MAPVALUES-0000000018 + <-- KSTREAM-SOURCE-0000000016 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-MAPVALUES-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000019 + <-- KTABLE-SOURCE-0000000017 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasL_Right + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasL_Left + Processor: KTABLE-TRANSFORMVALUES-0000000019 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000018 + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-JOINTHIS-0000000022 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000023 + <-- KTABLE-TRANSFORMVALUES-0000000019 + Processor: KTABLE-JOINOTHER-0000000023 (stores: [KafkaTopic_L_Left-Reduce, KafkaTopic_L_Right-Reduce]) + --> KTABLE-MERGE-0000000021 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000022 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000021 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-MERGE-0000000021 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000022, KTABLE-JOINOTHER-0000000023 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000025 + <-- KTABLE-MERGE-0000000021 + Processor: KTABLE-TOSTREAM-0000000025 (stores: []) + --> KSTREAM-SINK-0000000026 + <-- Project + Sink: KSTREAM-SINK-0000000026 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000025 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/plan.json new file mode 100644 index 000000000000..60febe92bd74 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T3.ID T3_ID,\n S1.V0 S1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN T2 T2 ON ((S1.ID = T2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T3_ID` INTEGER KEY, `S1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "S1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "T3_ID" + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/spec.json new file mode 100644 index 000000000000..147a4219319e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/spec.json @@ -0,0 +1,140 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164296795, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.L_Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "third join column in projection", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT T3.ID, s1.V0, t2.V0, t3.V0 FROM S1 JOIN T2 ON S1.ID = T2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "stream", + "schema" : "T3_ID INT KEY, S1_V0 BIGINT, T2_V0 BIGINT, T3_V0 BIGINT" + } ], + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right2", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/topology new file mode 100644 index 000000000000..5de3d9d215c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_third_join_column_in_projection/6.1.0_1594164296795/topology @@ -0,0 +1,50 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-SOURCE-0000000012 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right]) + --> KTABLE-SOURCE-0000000008 + Processor: PrependAliasL_Left (stores: []) + --> L_Join + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: L_Join (stores: [KafkaTopic_L_Right-Reduce]) + --> Join + <-- PrependAliasL_Left + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- L_Join + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000018 + <-- Join + Sink: KSTREAM-SINK-0000000018 (topic: OUTPUT) + <-- Project + Processor: PrependAliasL_Right (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/plan.json new file mode 100644 index 000000000000..c48378395144 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/plan.json @@ -0,0 +1,130 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (`some.key` STRING PRIMARY KEY, V STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`some.key` STRING KEY, `V` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT *\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`some.key` STRING KEY, `V` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`some.key` STRING KEY, `V` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "some.key" ], + "selectExpressions" : [ "V AS V" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/spec.json new file mode 100644 index 000000000000..eab2087e142a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/spec.json @@ -0,0 +1,73 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164303934, + "path" : "query-validation-tests/quoted-identifiers.json", + "schemas" : { + "CTAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "testCase" : { + "name" : "create table with key that is quoted", + "inputs" : [ { + "topic" : "test_topic", + "key" : "a", + "value" : { + "v" : "key" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a", + "value" : { + "V" : "key" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (`some.key` STRING PRIMARY KEY, v VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT * FROM TEST;" ], + "post" : { + "topics" : { + "topics" : [ { + "name" : "OUTPUT", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "formatInfo" : { + "format" : "KAFKA" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/topology new file mode 100644 index 000000000000..5f682997c6b5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_create_table_with_key_that_is_quoted/6.1.0_1594164303934/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Project + <-- KTABLE-MAPVALUES-0000000003 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Project + Sink: KSTREAM-SINK-0000000007 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/plan.json new file mode 100644 index 000000000000..a8189fc6844c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM L (`the key` STRING KEY, `SELECT` STRING, `field!` STRING) WITH (KAFKA_TOPIC='left_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "L", + "schema" : "`the key` STRING KEY, `SELECT` STRING, `field!` STRING", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE R (`with.dot` STRING PRIMARY KEY, `field 0` STRING) WITH (KAFKA_TOPIC='right_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "R", + "schema" : "`with.dot` STRING KEY, `field 0` STRING", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM JOINED AS SELECT *\nFROM L L\nLEFT OUTER JOIN R R ON ((L.`SELECT` = R.`with.dot`))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "JOINED", + "schema" : "`L_SELECT` STRING KEY, `L_the key` STRING, `L_field!` STRING, `R_with.dot` STRING, `R_field 0` STRING", + "topicName" : "JOINED", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "JOINED", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "JOINED" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`the key` STRING KEY, `SELECT` STRING, `field!` STRING" + }, + "keyExpression" : "`SELECT`" + }, + "keyColumnNames" : [ "L_SELECT" ], + "selectExpressions" : [ "`SELECT` AS L_SELECT", "`field!` AS `L_field!`", "ROWTIME AS L_ROWTIME", "`the key` AS `L_the key`" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`with.dot` STRING KEY, `field 0` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "R_with.dot" ], + "selectExpressions" : [ "`field 0` AS `R_field 0`", "ROWTIME AS R_ROWTIME", "`with.dot` AS `R_with.dot`" ] + }, + "keyColName" : "L_SELECT" + }, + "keyColumnNames" : [ "L_SELECT" ], + "selectExpressions" : [ "`L_the key` AS `L_the key`", "`L_field!` AS `L_field!`", "`R_with.dot` AS `R_with.dot`", "`R_field 0` AS `R_field 0`" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "JOINED" + }, + "queryId" : "CSAS_JOINED_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.error.classifier.regex" : "" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/spec.json new file mode 100644 index 000000000000..5c7403caa564 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/quoted-identifiers_-_joins_using_fields_that_require_quotes/6.1.0_1594164304009/spec.json @@ -0,0 +1,116 @@ +{ + "version" : "6.1.0", + "timestamp" : 1594164304009, + "path" : "query-validation-tests/quoted-identifiers.json", + "schemas" : { + "CSAS_JOINED_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_JOINED_0.KafkaTopic_Left.Source" : "STRUCT