gcloud_cluster_setup.txt

##### To create a single-node cluster #####
gcloud beta dataproc clusters create cluster-0098 \
	--region us-east1 \
	--zone us-east1-b \  
	--single-node \ 
	--master-machine-type n1-standard-4 \
	--master-boot-disk-size 2000 \
	--image-version 1.5-debian10 \ 
	--enable-component-gateway \
	--optional-components ANACONDA,JUPYTER \
	--bucket saberbf0098 \
	--properties dataproc:dataproc.logging.stackdriver.job.driver.enable=true,dataproc:dataproc.logging.stackdriver.enable=true,dataproc:jobs.file-backed-output.enable=true,dataproc:dataproc.logging.stackdriver.job.yarn.container.enable=true,yarn:yarn.log-aggregation-enable=true,yarn:yarn.nodemanager.remote-app-log-dir=gs://saberbf0098/logs,yarn:yarn.log-aggregation.retain-seconds=-1 \
	--scopes 'https://www.googleapis.com/auth/cloud-platform' \
	--metadata 'PIP_PACKAGES=google-cloud-bigquery google-cloud-storage' \
	--initialization-actions gs://goog-dataproc-initialization-actions-us-east1/python/pip-install.sh \    
	--project bigdata-09012020


##### To create a master-worker cluster #####
gcloud beta dataproc clusters create cluster-0098-mw \
    --region us-east1 \
    --zone us-east1-b \
    --master-machine-type n1-standard-4 \
    --master-boot-disk-type pd-ssd \
    --master-boot-disk-size 250 \
    --num-master-local-ssds 1 \
    --num-workers 2 \
    --worker-machine-type n1-standard-4 \
    --worker-boot-disk-type pd-ssd \
    --worker-boot-disk-size 125 \
    --num-worker-local-ssds 1 \
    --image-version 1.5-debian10 \
    --enable-component-gateway \
    --optional-components ANACONDA,JUPYTER \
    --bucket saberbf0098 \
    --scopes 'https://www.googleapis.com/auth/cloud-platform' \
    --metadata 'PIP_PACKAGES=google-cloud-bigquery google-cloud-storage' \
    --initialization-actions gs://goog-dataproc-initialization-actions-us-east1/python/pip-install.sh \
    --project bigdata-09012020