forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run-ab-platform.sh
executable file
·412 lines (366 loc) · 13.9 KB
/
run-ab-platform.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
#!/usr/bin/env bash
VERSION=0.63.19
# Run away from anything even a little scary
set -o nounset # -u exit if a variable is not set
set -o errexit # -f exit for any command failure"
readonly scriptVersion="$VERSION"
# text color escape codes (please note \033 == \e but OSX doesn't respect the \e)
blue_text='\033[94m'
red_text='\033[31m'
default_text='\033[0m'
# set -x/xtrace uses a Sony PS4 for more info
PS4="$blue_text""${0}:${LINENO}: ""$default_text"
############################################################
# Help #
############################################################
Help()
{
# Display Help
echo -e "This Script will download the necessary files for running docker compose"
echo -e "It will also run docker compose up"
echo -e "Take Warning! These assets may become stale over time!"
echo
# $0 is the currently running program
echo -e "Syntax: $0"
echo -e "options:"
echo -e " -d --download Only download files - don't run docker compose"
echo -e " -r --refresh ${red_text}DELETE${default_text} existing assets and re-download new ones"
echo -e " -h --help Print this Help."
echo -e " -x --debug Verbose mode."
echo -e " -b --background Run docker compose up in detached mode."
echo -e " --dnt Disable telemetry collection"
echo -e ""
}
########## Declare assets care about ##########
docker_compose_yaml="docker-compose.yaml"
docker_compose_debug_yaml="docker-compose.debug.yaml"
dot_env=".env"
dot_env_dev=".env.dev"
flags="flags.yml"
temporal_yaml="temporal/dynamicconfig/development.yaml"
# any string is an array to POSIX shell. Space separates values
all_files="$docker_compose_yaml $docker_compose_debug_yaml $dot_env $dot_env_dev $flags $temporal_yaml"
# Pinning the version this supports at v0.63.13, as docker compose is being deprecated.
# This has been pinned here as the VERSION variable (found above) is automatically updated as newer versions of Airbyte are released.
base_github_url="https://raw.githubusercontent.com/airbytehq/airbyte-platform/v0.63.13/"
# event states are used for telemetry data
readonly eventStateStarted="started"
readonly eventStateFailed="failed"
readonly eventStateSuccess="succeeded"
# event types are used for telemetry data
readonly eventTypeDownload="download"
readonly eventTypeInstall="install"
readonly eventTypeRefresh="refresh"
readonly eventTypeUninstall="uninstall"
telemetrySuccess=false
telemetrySessionUUID=""
# Deprecated, use telemetryUserUUID instead
# only here for backwards compatability reasons, can be removed when the following issues have been resolved:
# - https://github.com/airbytehq/airbyte-internal-issues/issues/7758
# - https://github.com/airbytehq/PyAirbyte/issues/219
telemetryUserULID=""
telemetryUserUUID=""
telemetryEnabled=true
# telemetry requires curl to be installed
if ! command -v curl > /dev/null; then
telemetryEnabled=false
fi
# TelemetryUUID echos a uuid value, or an empty string if no uuid can be genereated
# Attempts to generate a uuid via the following methods: uuidgen, /proc/sys/kernel/random/uuid, openssl, python
TelemetryUUID() {
local uuid
if command -v uuidgen > /dev/null; then
uuid=$(uuidgen)
elif [ -f /proc/sys/kernel/random/uuid ]; then
uuid=$(cat /proc/sys/kernel/random/uuid)
elif command -v openssl > /dev/null; then
local opensslUUID
opensslUUID=$(openssl rand -hex 16)
uuid=${opensslUUID:0:8}-${opensslUUID:8:4}-${opensslUUID:12:4}-${opensslUUID:16:4}-${opensslUUID:20:12}
elif command -v python > /dev/null; then
uuid=$(python -c "import uuid; print(uuid.uuid4())")
else
uuid=""
fi
echo $uuid
}
# TelemetryConfig configures the telemetry variables and will disable telemetry if it cannot be configured.
TelemetryConfig()
{
# only attempt to do anything if telemetry is not disabled
if $telemetryEnabled; then
telemetrySessionUUID=$(TelemetryUUID)
if [[ $telemetrySessionUUID = "" ]]; then
# if we still don't have a uuid, give up on telemetry data
telemetryEnabled=false
return
fi
# if we have an analytics file, use it
if test -f ~/.airbyte/analytics.yml; then
# grab the deprecated ulid value, as we may still write it to the file if we write the file with a uuid
telemetryUserULID=$(cat ~/.airbyte/analytics.yml | grep "anonymous_user_id" | cut -d ":" -f2 | xargs)
telemetryUserUUID=$(cat ~/.airbyte/analytics.yml | grep "analytics_id" | cut -d ":" -f2 | xargs)
fi
# if the telemetery uuid is still undefined, attempt to create it and write the analytics file
if [[ $telemetryUserUUID = "" ]] ; then
telemetryUserUUID=$(TelemetryUUID)
if [[ $telemetryUserUUID = "" ]]; then
# if we still don't have a uuid, give up on telemetry data
telemetryEnabled=false
else
# we created a new uuid, write it out
echo "Thanks you for using Airbyte!"
echo "Anonymous usage reporting is currently enabled. For more information, please see https://docs.airbyte.com/telemetry"
mkdir -p ~/.airbyte
cat > ~/.airbyte/analytics.yml <<EOL
# This file is used by Airbyte to track anonymous usage statistics.
# For more information or to opt out, please see
# - https://docs.airbyte.com/operator-guides/telemetry
# Deprecated, use analytics_id instead
anonymous_user_id: $telemetryUserULID
analytics_id: $telemetryUserUUID
EOL
fi
fi
fi
}
# TelemetryDockerUp checks if the webapp container is in a running state. If it is it will send a successful event.
# if after 20 minutes it hasn't succeeded, a failed event will be sent (or if the user terminates early, a failed event would
# also be sent).
#
# Note this only checks if the webapp container is running, that doesn't actually mean the entire stack is up.
# Some further refinement on this metric may be necessary.
TelemetryDockerUp()
{
if ! $telemetryEnabled; then
return
fi
# for up to 1200 seconds (20 minutes), check to see if the server services is in a running state
end=$((SECONDS+1200))
while [ $SECONDS -lt $end ]; do
webappState=$(docker compose ps --all --format "{{.Service}}:{{.State}}" 2>/dev/null | grep ^server | cut -d ":" -f2 | xargs)
if [ "$webappState" = "running" ]; then
TelemetrySend $eventStateSuccess $eventTypeInstall
return
fi
sleep 1
done
TelemetrySend $eventStateFailed $eventTypeInstall "webapp was not running within 1200 seconds"
}
readonly telemetryKey="kpYsVGLgxEqD5OuSZAQ9zWmdgBlyiaej"
readonly telemetryURL="https://api.segment.io/v1/track"
TelemetrySendTrap()
{
if $telemetrySuccess; then
# due to how traps work, we don't want to send a failure for exiting docker after we sent a success
return
fi
# start, failed, success
local state=$1
# install, uninstall
local event=$2
# optional error
local err=${3:-""}
TelemetrySend "$state" "$event" "$err"
}
TelemetrySend()
{
if $telemetryEnabled; then
# start, failed, success
local state=$1
# install, uninstall
local event=$2
# optional error
local err=${3:-""}
local now=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
local body=$(cat << EOL
{
"anonymousId":"$telemetryUserUUID",
"event":"$event",
"properties": {
"deployment_method":"run_ab",
"session_id":"$telemetrySessionUUID",
"state":"$state",
"os":"$OSTYPE",
"script_version":"$scriptVersion",
"error":"$err"
},
"timestamp":"$now",
"writeKey":"$telemetryKey"
}
EOL
)
curl -s -o /dev/null -H "Content-Type: application/json" -X POST -d "$body" $telemetryURL
if [[ $state = "success" ]]; then {
telemetrySuccess=true
}
fi
fi
}
TelemetryConfig
############################################################
# Download #
############################################################
Download()
{
########## Check if we already have the assets we are looking for ##########
for file in $all_files; do
# Account for the case where the file is in a subdirectory.
# Make sure the directory exists to keep curl happy.
dir_path=$(dirname "${file}")
mkdir -p "${dir_path}"
if test -f $file; then
# Check if the assets are old. A possibly sharp corner
if test $(find $file -type f -mtime +60 > /dev/null); then
echo -e "$red_text""Warning your $file may be stale!""$default_text"
echo -e "$red_text""rm $file to refresh!""$default_text"
else
echo -e "$blue_text""found $file locally!""$default_text"
fi
else
echo -e "$blue_text""Downloading $file""$default_text"
curl --location\
--fail\
--silent\
--show-error \
${base_github_url}${file} > $file
fi
done
}
DeleteLocalAssets()
{
for file in $all_files; do
echo -e "$blue_text""Attempting to delete $file!""$default_text"
if test -f $file; then
rm $file && echo -e "It's gone!"
else
echo -e "$file not found locally. Nothing to delete."
fi
done
}
dockerDetachedMode=""
# $0 is the currently running program (this file)
this_file_directory=$(dirname $0)
# Run this from the / directory because we assume relative paths
cd ${this_file_directory}
args=$@
# Parse the arguments for specific flags before parsing for actions.
for argument in $args; do
case $argument in
-h | --help)
Help
exit
;;
-b | --background)
dockerDetachedMode="-d"
;;
--dnt)
telemetryEnabled=false
;;
esac
done
for argument in $args; do
case $argument in
-d | --download)
TelemetrySend $eventStateStarted $eventTypeDownload
trap 'TelemetrySendTrap $eventStateFailed $eventTypeDownload "sigint"' SIGINT
trap 'TelemetrySendTrap $eventStateFailed $eventTypeDownload "sigterm"' SIGTERM
Download
TelemetrySend $eventStateSuccess $eventTypeDownload
exit
;;
-r | --refresh)
TelemetrySend $eventStateStarted $eventTypeRefresh
trap 'TelemetrySendTrap $eventStateFailed $eventTypeRefresh "sigint"' SIGINT
trap 'TelemetrySendTrap $eventStateFailed $eventTypeRefresh "sigterm"' SIGTERM
DeleteLocalAssets
Download
TelemetrySend $eventStateSuccess $eventTypeRefresh
exit
;;
-x | --debug)
set -o xtrace # -x display every line before execution; enables PS4
;;
-h | --help)
# noop, this was checked in the previous for loop
;;
-b | --background)
# noop, this was checked in the previous for loop
;;
--dnt)
# noop, this was checked in the previous for loop
;;
*)
echo "$argument is not a known command."
echo
Help
exit
;;
esac
done
TelemetrySend $eventStateStarted $eventTypeInstall
trap 'TelemetrySendTrap $eventStateFailed $eventTypeInstall "sigint"' SIGINT
trap 'TelemetrySendTrap $eventStateFailed $eventTypeInstall "sigterm"' SIGTERM
########## Pointless Banner for street cred ##########
# Make sure the console is huuuge
if test $(tput cols) -ge 64; then
# Make it green!
echo -e "\033[32m"
echo -e " █████╗ ██╗██████╗ ██████╗ ██╗ ██╗████████╗███████╗"
echo -e "██╔══██╗██║██╔══██╗██╔══██╗╚██╗ ██╔╝╚══██╔══╝██╔════╝"
echo -e "███████║██║██████╔╝██████╔╝ ╚████╔╝ ██║ █████╗ "
echo -e "██╔══██║██║██╔══██╗██╔══██╗ ╚██╔╝ ██║ ██╔══╝ "
echo -e "██║ ██║██║██║ ██║██████╔╝ ██║ ██║ ███████╗"
echo -e "╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝ ╚══════╝"
echo -e " Move Data"
# Make it less green
echo -e "\033[0m"
sleep 1
fi
deprecation_text='\033[1;91m'
echo -e "$deprecation_text"
echo -e "*** Deprecated Warning!"
echo -e "* This script, and docker compose support, have both been deprecated."
echo -e "*"
echo -e "* For additional information please visit the following:"
echo -e "* https://docs.airbyte.com/using-airbyte/getting-started/oss-quickstart"
echo -e "$default_text"
sleep 2
########## Dependency Check ##########
if ! docker compose version >/dev/null 2>/dev/null; then
echo -e "$red_text""docker compose v2 not found! please install docker compose!""$default_text"
TelemetrySend $eventStateFailed $eventTypeInstall "docker compose not installed"
exit 1
fi
Download
########## Source Environmental Variables ##########
for file in $dot_env $dot_env_dev; do
echo -e "$blue_text""Loading Shell Variables from $file...""$default_text"
source $file
done
########## Start Docker ##########
echo
echo -e "$blue_text""Starting Docker Compose""$default_text"
if [ -z "$dockerDetachedMode" ]; then
# if running in docker-detach mode, kick off a background task as `docker compose up` will be a blocking
# call and we'll have no way to determine when we've successfully started.
TelemetryDockerUp &
fi
AIRBYTE_INSTALLATION_ID="${telemetryUserUUID}" docker compose up $dockerDetachedMode
# $? is the exit code of the last command. So here: docker compose up
if test $? -ne 0; then
echo -e "$red_text""Docker compose failed. If you are seeing container conflicts""$default_text"
echo -e "$red_text""please consider removing old containers""$default_text"
TelemetrySend $eventStateFailed $eventTypeInstall "docker compose failed"
else
if [ -z "$dockerDetachedMode" ]; then
# not running in detached mode
TelemetrySend $eventStateSuccess $eventTypeUninstall
fi
fi
########## Ending Docker ##########
if [ -z "$dockerDetachedMode" ]; then
docker compose down
else
echo -e "$blue_text""Airbyte containers are running!""$default_text"
fi