Merge pull request #4556 from communitybridge/unicron-cla-authorization

Added utils/investigation scripts

utils/check_unique.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# PROD=1
+# LIM=10
+# DEBUG=1
+if [ -z "$LIM" ]
+then
+  lim=1
+else
+  lim=$LIM
+fi
+if [ -z "$PROD" ]
+then
+  schema='dynamodb_product_us_east1_dev'
+  prefix='cla_dev_'
+else
+  schema='dynamodb_product_us_east_1'
+  prefix='cla_prod_'
+fi
+if [ -z "$1" ]
+then
+  echo "you need to specify table to query as a 1st parameter, for example 'signatures'"
+  echo "possible tables include: approvals, ccla_whitelist_requests, cla_manager_requests, companies, company_invites, events, gerrit_instances, github_orgs, gitlab_orgs, metrics, projects, projects_cla_groups, repositories, session_store, signatures, store, user_permissions, users"
+  exit 1
+fi
+if [ -z "$2" ]
+then
+  echo "$0: required argument - list of columns to check for uniqueness on $1, example: 'lf_username user_company_id'"
+  exit 2
+fi
+query="select"
+ary=($2)
+n=0
+for c in "${ary[@]}"
+do
+  if [ -z "${cols}" ]
+  then
+    cols="data:${c}"
+    cond="where ${cols} is not null"
+  else
+    cols="${cols}, data:${c}"
+    cond="${cond} and data:${c} is not null"
+  fi
+done
+query="${query} ${cols}, count(*) as cnt from fivetran_ingest.${schema}.${prefix}${1} ${cond} group by all"
+query="select i.* from (${query}) i where i.cnt > 1 order by i.cnt desc limit ${lim}"
+if [ ! -z "$DEBUG" ]
+then
+  echo "query: ${query}"
+fi
+snowsql $(cat ./snowflake.secret) -o friendly=false -o header=true -o timing=false -o output_format=plain -q "${query}"

utils/cla_authorization.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# API_URL=https://[xyz].ngrok-free.app (defaults to localhost:5000)
+# cla_group='b71c469a-55e7-492c-9235-fd30b31da2aa' (ONAP)
+# lfid='lflgryglicki'
+# TOKEN='...' - Auth0 JWT bearer token
+# XACL='...' - X-ACL
+# API_URL='https://api-gw.platform.linuxfoundation.org/cla-service/v4/cla-service/v4/'
+# DEBUG=1 ./utils/cla_group_corporate_contributors.sh b71c469a-55e7-492c-9235-fd30b31da2aa andreasgeissler
+
+if [ -z "$TOKEN" ]
+then
+  # source ./auth0_token.secret
+  TOKEN="$(cat ./auth0.token.secret)"
+fi
+
+if [ -z "$TOKEN" ]
+then
+  echo "$0: TOKEN not specified and unable to obtain one"
+  exit 1
+fi
+
+if [ -z "$XACL" ]
+then
+  XACL="$(cat ./x-acl.secret)"
+fi
+
+if [ -z "$XACL" ]
+then
+  echo "$0: XACL not specified and unable to obtain one"
+  exit 2
+fi
+
+if [ -z "$1" ]
+then
+  echo "$0: you need to specify cla_group UUID as a 1st parameter, example: 'b71c469a-55e7-492c-9235-fd30b31da2aa', '01af041c-fa69-4052-a23c-fb8c1d3bef24'"
+  exit 3
+fi
+export cla_group="$1"
+
+if [ -z "$2" ]
+then
+  echo "$0: you need to specify lfid as a 2nd parameter, example: 'andreasgeissler'"
+  exit 4
+fi
+export lfid="$2"
+
+if [ -z "$API_URL" ]
+then
+  export API_URL="http://localhost:5000"
+fi
+
+if [ ! -z "$DEBUG" ]
+then
+  echo "curl -s -XGET -H 'X-ACL: ${XACL}' -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' '${API_URL}/v4/cla-services/cla/authorization?lfid=${lfid}&claGroupId=${cla_group}'"
+fi
+curl -s -XGET -H "X-ACL: ${XACL}" -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" "${API_URL}/v4/cla/authorization?lfid=${lfid}&claGroupId=${cla_group}"

utils/lookup_cla_group_sf.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+if [ -z "$1" ]
+then
+  echo "$0: you need to specify which column as a 1st parameter, for example: 'cla_group_name'"
+  echo "possible columns include: project_sfid, cla_group_id, cla_group_name, date_created, date_modified, foundation_name, foundation_sfid, note, project_external_id, project_name, repositories_count, version"
+  exit 1
+fi
+if [ -z "$2" ]
+then
+  echo "$0: you need to specify '${1}' value as a 2nd parameter, for example: 'onap'"
+  exit 2
+fi
+snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select object_construct('project_sfid', project_sfid, 'data', data) from fivetran_ingest.DYNAMODB_PRODUCT_US_EAST_1.CLA_PROD_PROJECTS_CLA_GROUPS where lower(data:${1}) = lower('${2}')" | jq -r '.'

utils/lookup_examples.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+./utils/lookup_sf.sh gerrit_instances gerrit_id project_sfid "'a09P000000DsCE5IAN'"
+./utils/lookup_sf.sh projects_cla_groups project_sfid project_sfid "'a09P000000DsCE5IAN'"
+OP=in ./utils/lookup_sf.sh user_permissions username projects "'a09P000000DsCE5IAN'"
+./utils/lookup_sf.sh signatures signature_id signature_type "'ecla'"
+./utils/lookup_sf.sh signatures signature_id project_id "'01af041c-fa69-4052-a23c-fb8c1d3bef24'"
+./utils/cla_authorization.sh 01af041c-fa69-4052-a23c-fb8c1d3bef24 poojapanjwani
+./utils/lookup_sf.sh projects_cla_groups project_sfid cla_group_id "'01af041c-fa69-4052-a23c-fb8c1d3bef24'"
+./utils/lookup_sf.sh companies company_id company_id "'f7c7ac9c-4dbf-4104-ab3f-6b38a26d82dc'"
+OUT='project_name' ./utils/lookup_sf.sh projects project_id project_id "'01af041c-fa69-4052-a23c-fb8c1d3bef24'"
+OUT='project_sfid project_name foundation_sfid' ./utils/lookup_sf.sh projects_cla_groups project_sfid foundation_sfid "'a09P000000DsCE5IAN'"
+# ICLA
+COND="data:signature_project_id = '01af041c-fa69-4052-a23c-fb8c1d3bef24' and data:signature_reference_id = '1527f0ec-3272-11ec-a3ed-0e7521e28b4e' and data:signature_type = 'cla' and data:signature_reference_type = 'user' and data:signature_user_ccla_company_id is null and data:signature_signed and data:signature_approved" ./utils/lookup_sf.sh signatures signature_id
+# ECLA
+COND="data:signature_reference_id = '1527f0ec-3272-11ec-a3ed-0e7521e28b4e' and data:signature_user_ccla_company_id = 'f7c7ac9c-4dbf-4104-ab3f-6b38a26d82dc' and data:signature_project_id = '01af041c-fa69-4052-a23c-fb8c1d3bef24'" ./utils/lookup_sf.sh signatures signature_id
+# CCLA
+COND="data:signature_project_id = '01af041c-fa69-4052-a23c-fb8c1d3bef24' and data:signature_reference_id = 'f7c7ac9c-4dbf-4104-ab3f-6b38a26d82dc' and data:signature_type = 'ccla' and data:signature_reference_type = 'company' and data:signature_user_ccla_company_id is null and data:signature_approved and data:signature_signed" ./utils/lookup_sf.sh signatures signature_id

utils/lookup_sf.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# PROD=1
+# DEBUG=1
+# FUNC=lower
+# OP=in
+# COND='complex expression'
+# OUT='col1 col 2 col3'
+# example:
+# DEBUG=1 PROD=1 FUNC=lower ./utils/lookup_sf.sh projects_cla_groups project_sfid cla_group_name "lower('onap')"
+# OUT='project_sfid project_name foundation_sfid' ./utils/lookup_sf.sh projects_cla_groups project_sfid foundation_sfid "'a09P000000DsCE5IAN'"
+
+if [ -z "$PROD" ]
+then
+  schema='dynamodb_product_us_east1_dev'
+  prefix='cla_dev_'
+else
+  schema='dynamodb_product_us_east_1'
+  prefix='cla_prod_'
+fi
+if [ -z "$1" ]
+then
+  echo "you need to specify table to query as a 1st parameter, for example 'signatures'"
+  echo "possible tables include: approvals, ccla_whitelist_requests, cla_manager_requests, companies, company_invites, events, gerrit_instances, github_orgs, gitlab_orgs, metrics, projects, projects_cla_groups, repositories, session_store, signatures, store, user_permissions, users"
+  exit 1
+fi
+if [ -z "$2" ]
+then
+  echo "you need to specify table's primary key column name as a 2nd parameter, see example row from the table to determine one:"
+  snowsql $(cat ./snowflake.secret) -o friendly=false -o header=true -o timing=false -o output_format=plain -q "select * from fivetran_ingest.${schema}.${prefix}${1} limit 1"
+  exit 2
+fi
+if [ -z "$COND" ]
+then
+  if [ -z "$3" ]
+  then
+    echo "$0: you need to specify $1 column as a 3rd parameter, see available columns to choose one:"
+    snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select object_construct('${2}', ${2}, 'data', data) from fivetran_ingest.${schema}.${prefix}${1} limit 1"
+    exit 3
+  fi
+  if [ -z "$4" ]
+  then
+    echo "$0: you need to specify ${1} ${3} value as a 4th parameter"
+    echo "$0: if that column is strinf then you need to specify like this: \"'value'\""
+    exit 4
+  fi
+  if [ "$3" = "$2" ]
+  then
+    col="$2"
+  else
+    col="data:${3}"
+  fi
+  if [ ! -z "$FUNC" ]
+  then
+    col="${FUNC}(${col})"
+  fi
+  if [ "$OP" = "in" ]
+  then
+    cond="array_contains(${4}::variant, ${col})"
+  else
+    cond="${col} = ${4}"
+  fi
+else
+  cond="${COND}"
+fi
+if [ ! -z "$DEBUG" ]
+then
+  echo "snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q \"select object_construct('${2}', ${2}, 'data', data) from fivetran_ingest.${schema}.${prefix}${1} where ${cond}\""
+fi
+if [ ! -z "$OUT" ]
+then
+  cols=($OUT)
+  n=0
+  for c in "${cols[@]}"
+  do
+    if [ "$c" = "$2" ]
+    then
+      cc=".${c}"
+    else
+      cc=".data.${c}"
+    fi
+    if [ -z "${jqq}" ]
+    then
+      jqq="${cc}"
+    else
+      jqq="${jqq},${cc}"
+    fi
+    n=$((n + 1))
+  done
+  if [[ $n -gt 1 ]]
+  then
+    jqq="[${jqq}]"
+  fi
+  snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select object_construct('${2}', ${2}, 'data', data) from fivetran_ingest.${schema}.${prefix}${1} where ${cond}" | jq -r "${jqq}"
+else
+  snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select object_construct('${2}', ${2}, 'data', data) from fivetran_ingest.${schema}.${prefix}${1} where ${cond}" | jq -r '.'
+fi

utils/lookup_user_by_lfid.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+if [ -z "$1" ]
+then
+  echo "$0: you need to specify lfid as a 1st parameter, for example: 'lgryglicki'"
+  exit 1
+fi
+aws --profile lfproduct-prod dynamodb query --table-name cla-prod-users --index-name lf-username-index --key-condition-expression "lf_username = :name" --expression-attribute-values "{\":name\":{\"S\":\"${1}\"}}" | jq -r '.'

utils/lookup_user_by_lfid_sf.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+if [ -z "$1" ]
+then
+  echo "$0: you need to specify lfid as a 1st parameter, for example: 'lgryglicki'"
+  exit 1
+fi
+snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select object_construct('user_id', user_id, 'data', data) from fivetran_ingest.DYNAMODB_PRODUCT_US_EAST_1.CLA_PROD_USERS where lower(data:lf_username) = lower('${1}')" | jq -r '.'

utils/lookup_user_sf.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+if [ -z "$1" ]
+then
+  echo "$0: you need to specify which column as a 1st parameter, for example: 'lf_username'"
+  echo "possible columns include: user_id, admin, date_created, date_modified, lf_email, lf_username, note, user_company_id, user_emails, user_external_id, user_github_id, user_github_username, user_name, version"
+  exit 1
+fi
+if [ -z "$2" ]
+then
+  echo "$0: you need to specify '${1}' value as a 2nd parameter, for example: 'lgryglicki'"
+  exit 2
+fi
+snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select object_construct('user_id', user_id, 'data', data) from fivetran_ingest.dynamodb_product_us_east_1.cla_prod_users where lower(data:${1}) = lower('${2}')" | jq -r '.'