diff --git a/.jenkins/pipelines/code-coverage.Jenkinsfile b/.jenkins/pipelines/code-coverage.Jenkinsfile index d76be61ce..084ba850e 100644 --- a/.jenkins/pipelines/code-coverage.Jenkinsfile +++ b/.jenkins/pipelines/code-coverage.Jenkinsfile @@ -14,7 +14,6 @@ pipeline { parameters { string(name: 'REPOSITORY', defaultValue: 'deislabs/mystikos') string(name: 'BRANCH', defaultValue: 'main', description: 'Branch to build') - choice(name: 'REGION', choices:['useast', 'canadacentral'], description: 'Azure region for SQL solutions test') string(name: 'PULL_REQUEST_ID', defaultValue: '', description: 'If you are building a pull request, enter the pull request ID number here. (ex. 789)') string(name: 'PACKAGE_NAME', defaultValue: '', description: 'optional - release package to install (do not include extension)') choice(name: 'PACKAGE_EXTENSION', choices:['tar.gz', 'deb'], description: 'Extension of package given in PACKAGE_NAME') @@ -75,8 +74,8 @@ pipeline { steps { withCredentials([string(credentialsId: 'Jenkins-ServicePrincipal-ID', variable: 'SERVICE_PRINCIPAL_ID'), string(credentialsId: 'Jenkins-ServicePrincipal-Password', variable: 'SERVICE_PRINCIPAL_PASSWORD'), - string(credentialsId: 'ACC-Prod-Tenant-ID', variable: 'TENANT_ID'), - string(credentialsId: 'ACC-Prod-Subscription-ID', variable: 'AZURE_SUBSCRIPTION_ID')]) { + string(credentialsId: 'Jenkins-CI-Tenant-Id', variable: 'TENANT_ID'), + string(credentialsId: 'Jenkins-CI-Subscription-Id', variable: 'AZURE_SUBSCRIPTION_ID')]) { sh """ ${JENKINS_SCRIPTS}/azure-sdk/install-azure-cli.sh ${JENKINS_SCRIPTS}/azure-sdk/login-azure-cli.sh @@ -102,21 +101,13 @@ pipeline { stage('Run Solutions Tests') { steps { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - withCredentials([string(credentialsId: "mystikos-sql-db-name-${REGION}", variable: 'DB_NAME'), - string(credentialsId: "mystikos-sql-db-server-name-${REGION}", variable: 'DB_SERVER_NAME'), - string(credentialsId: "mystikos-maa-url-${REGION}", variable: 'MAA_URL'), - string(credentialsId: 'mystikos-sql-db-userid', variable: 'DB_USERID'), - string(credentialsId: 'mystikos-sql-db-password', variable: 'DB_PASSWORD'), - string(credentialsId: 'mystikos-mhsm-client-secret', variable: 'CLIENT_SECRET'), - string(credentialsId: 'mystikos-mhsm-client-id', variable: 'CLIENT_ID'), - string(credentialsId: 'mystikos-mhsm-app-id', variable: 'APP_ID'), - string(credentialsId: 'mystikos-mhsm-aad-url', variable: 'MHSM_AAD_URL'), - string(credentialsId: 'mystikos-mhsm-ssr-pkey', variable: 'SSR_PKEY') + withCredentials([string(credentialsId: "mystikos-sql-db-name-useast2", variable: 'DB_NAME'), + string(credentialsId: "mystikos-sql-db-server-name-useast2", variable: 'DB_SERVER_NAME'), + string(credentialsId: "mystikos-maa-url-useast2", variable: 'MAA_URL') ]) { sh """ echo "MYST_NIGHTLY_TEST is set to \${MYST_NIGHTLY_TEST}" echo "MYST_SKIP_PR_TEST is set to \${MYST_SKIP_PR_TEST}" - echo "Running in ${REGION}" make solutions_tests echo "Running samples" sudo make install @@ -133,11 +124,11 @@ pipeline { steps { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { withCredentials([string(credentialsId: 'Jenkins-ServicePrincipal-ID', variable: 'servicePrincipalId'), - string(credentialsId: 'ACC-Prod-Tenant-ID', variable: 'tenantId'), string(credentialsId: 'Jenkins-ServicePrincipal-Password', variable: 'servicePrincipalKey'), string(credentialsId: 'mystikos-ci-keyvault-url', variable: 'AZURE_KEYVAULT_URL'), string(credentialsId: 'mystikos-ci-keyvault-url', variable: 'AZURE_TEST_KEYVAULT_URL'), - string(credentialsId: 'ACC-Prod-Subscription-ID', variable: 'AZURE_SUBSCRIPTION_ID'), + string(credentialsId: 'Jenkins-CI-Tenant-Id', variable: 'TENANT_ID'), + string(credentialsId: 'Jenkins-CI-Subscription-Id', variable: 'AZURE_SUBSCRIPTION_ID'), string(credentialsId: 'mystikos-storage-mystikosciacc-connectionstring', variable: 'STANDARD_STORAGE_CONNECTION_STRING')]) { sh """ ${JENKINS_SCRIPTS}/global/run-azure-tests.sh \ diff --git a/.jenkins/pipelines/standalone/solutions-tests.Jenkinsfile b/.jenkins/pipelines/standalone/solutions-tests.Jenkinsfile index cceb135fc..de8c08c72 100644 --- a/.jenkins/pipelines/standalone/solutions-tests.Jenkinsfile +++ b/.jenkins/pipelines/standalone/solutions-tests.Jenkinsfile @@ -11,7 +11,6 @@ pipeline { string(name: "REPOSITORY", defaultValue: "deislabs/mystikos") string(name: "BRANCH", defaultValue: "main", description: "Branch to build") string(name: "PULL_REQUEST_ID", defaultValue: "", description: "If you are building a pull request, enter the pull request ID number here. (ex. 789)") - choice(name: "REGION", choices: ['useast', 'canadacentral'], description: "Azure region for the SQL solution tests") choice(name: "TEST_CONFIG", choices: ['None', 'Nightly', 'Code Coverage'], description: "Test configuration to execute") choice(name: "VM_GENERATION", choices: ['v3', 'v2'], description: "v3 for Ice Lake VMs; v2 for Coffee Lake") string(name: "COMMIT_SYNC", defaultValue: "", description: "optional - used to sync outputs of parallel jobs") @@ -85,7 +84,23 @@ pipeline { # Install global dependencies ${JENKINS_SCRIPTS}/global/wait-dpkg.sh ${JENKINS_SCRIPTS}/global/init-install.sh + + ${JENKINS_SCRIPTS}/global/wait-dpkg.sh + sudo apt-get install -y jq + ${JENKINS_SCRIPTS}/azure-sdk/install-azure-cli.sh """ + // Export UAMI's client id to use in another stage + // This returns a JWT token from which we extract the appid + script { + sh 'az login --identity' + UAMI_ID = sh( + returnStdout: true, + script: ''' + az account get-access-token --query "accessToken" -o tsv | cut -d '.' -f 2 | base64 --decode | jq '.appid' | sed 's/"//g' + ''' + ).trim() + sh 'echo $UAMI_ID' + } } } } @@ -135,23 +150,23 @@ pipeline { stage('Run Solutions Tests') { steps { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - withCredentials([string(credentialsId: "mystikos-sql-db-name-${REGION}", variable: 'DB_NAME'), - string(credentialsId: "mystikos-sql-db-server-name-${REGION}", variable: 'DB_SERVER_NAME'), - string(credentialsId: "mystikos-maa-url-${REGION}", variable: 'MAA_URL'), - string(credentialsId: 'mystikos-sql-db-userid', variable: 'DB_USERID'), - string(credentialsId: 'mystikos-sql-db-password', variable: 'DB_PASSWORD') + withCredentials([string(credentialsId: "mystikos-sql-db-name-useast2", variable: 'DB_NAME'), + string(credentialsId: "mystikos-sql-db-server-name-useast2", variable: 'DB_SERVER_NAME'), + string(credentialsId: "mystikos-maa-url-useast2", variable: 'MAA_URL') ]) { - sh """ - echo "MYST_NIGHTLY_TEST is set to \${MYST_NIGHTLY_TEST}" - echo "MYST_SKIP_PR_TEST is set to \${MYST_SKIP_PR_TEST}" - echo "Running in ${REGION}" - make solutions_tests - echo "Running samples" - sudo make install - export PATH="/opt/mystikos/bin:$PATH" - export MYSTIKOS_INSTALL_DIR="/opt/mystikos/" - make -j -C ${WORKSPACE}/samples - """ + withEnv(["DB_USERID=${UAMI_ID}"]) { + sh """ + echo "DB_USERID is set to \${DB_USERID}" + echo "MYST_NIGHTLY_TEST is set to \${MYST_NIGHTLY_TEST}" + echo "MYST_SKIP_PR_TEST is set to \${MYST_SKIP_PR_TEST}" + make solutions_tests + echo "Running samples" + sudo make install + export PATH="/opt/mystikos/bin:$PATH" + export MYSTIKOS_INSTALL_DIR="/opt/mystikos/" + make -j -C ${WORKSPACE}/samples + """ + } } } } diff --git a/solutions/python_app/Dockerfile b/solutions/python_app/Dockerfile index 825e336fd..bf746f89c 100644 --- a/solutions/python_app/Dockerfile +++ b/solutions/python_app/Dockerfile @@ -5,7 +5,7 @@ RUN apt update && apt install -y curl gnupg2 &&\ curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list > /etc/apt/sources.list.d/mssql-release.list &&\ apt update && \ apt install -y wget &&\ - ACCEPT_EULA=Y apt install -y msodbcsql17 &&\ + ACCEPT_EULA=Y apt install -y msodbcsql18 &&\ wget -q https://repo.continuum.io/miniconda/Miniconda3-py38_4.8.2-Linux-x86_64.sh &&\ chmod 755 Miniconda3-py38_4.8.2-Linux-x86_64.sh &&\ ./Miniconda3-py38_4.8.2-Linux-x86_64.sh -b -p /miniconda &&\ diff --git a/solutions/python_app/app.py b/solutions/python_app/app.py index 5d7f7e5bb..3f36ca15c 100644 --- a/solutions/python_app/app.py +++ b/solutions/python_app/app.py @@ -9,12 +9,17 @@ from logzero import logger from Crypto.Cipher import AES -def test_pyodbc(server: str, database: str, uid: str, password: str, driver: str, query: str): +def test_pyodbc(server: str, database: str, driver: str, query: str, uid: str = '', password: str = ''): connstr = "Driver=" + driver connstr += ";Server=" + server connstr += ";Database=" + database - connstr += ";UID=" + uid - connstr += ";PWD=" + password + if uid: + connstr += ";User ID=" + uid + # Use password authentication if password is provided, otherwise use Managed Identity + if password: + connstr += ";PWD=" + password + else: + connstr += ";Authentication=ActiveDirectoryMsi" try: conn = pyodbc.connect(connstr) @@ -95,9 +100,9 @@ def test_bidirectional_pty(): test_pyodbc( server=os.getenv("DB_SERVER_NAME"), database=os.getenv("DB_NAME"), - uid = os.getenv("DB_USERID"), - password = os.getenv("DB_PASSWORD"), - driver="{ODBC Driver 17 for SQL Server}", + uid = os.getenv("DB_USERID", ""), + password = os.getenv("DB_PASSWORD", ""), + driver="{ODBC Driver 18 for SQL Server}", query="SELECT USER_NAME()") test_pandas() diff --git a/solutions/python_app/config.json b/solutions/python_app/config.json index b34e3ea7f..7b765703c 100644 --- a/solutions/python_app/config.json +++ b/solutions/python_app/config.json @@ -20,5 +20,6 @@ // The environment variables accessible inside the enclave. "EnvironmentVariables": [""], // The environment variables we get from the host - "HostEnvironmentVariables": ["DB_SERVER_NAME", "DB_NAME", "MAA_ENDPOINT", "DB_USERID", "DB_PASSWORD"] + "HostEnvironmentVariables": ["DB_SERVER_NAME", "DB_NAME", "MAA_ENDPOINT", "DB_USERID", "DB_PASSWORD"], + "ThreadStackSize": "1m" } diff --git a/solutions/sql_ae/Dockerfile b/solutions/sql_ae/Dockerfile index 3f6163b50..929775ed7 100644 --- a/solutions/sql_ae/Dockerfile +++ b/solutions/sql_ae/Dockerfile @@ -4,6 +4,6 @@ USER root RUN apk add --no-cache bash sudo unixodbc-dev curl krb5-libs libstdc++ mbedtls RUN mkdir -p /tmp/msodbcinstall && cd /tmp/msodbcinstall && \ -curl --retry 5 --retry-max-time 120 -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.6.1.1-1_amd64.apk +curl --retry 5 --retry-max-time 120 -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_18.3.3.1-1_amd64.apk RUN cd /tmp/msodbcinstall && sudo apk add --allow-untrusted $(ls) RUN rm -rf /tmp/msodbcinstall diff --git a/solutions/sql_ae/Makefile b/solutions/sql_ae/Makefile index 3aa7b4db0..27876c2d7 100644 --- a/solutions/sql_ae/Makefile +++ b/solutions/sql_ae/Makefile @@ -18,7 +18,7 @@ all: myst app $(MYST) mkcpio $(APPDIR) rootfs run: appdir - $(RUNTEST) $(MYST_EXEC) rootfs /odbc_app classic default $(OPTS) + $(RUNTEST) $(MYST_EXEC) rootfs /odbc_app msi default $(OPTS) myst: $(MAKE) -C $(TOP)/tools/myst @@ -26,7 +26,7 @@ myst: app: rm -rf app/odbc_app $(APPDIR) docker build -t sql_ae.alpine.build -f app-dockerfile . - docker run --rm -v $(CURDIR)/app:/app sql_ae.alpine.build bash -c "gcc -g -fshort-wchar -fPIC -o /app/cksp.so -shared /app/cksp.c -I/opt/microsoft/msodbcsql17/include; gcc -g -o /app/odbc_app -fshort-wchar /app/odbc_app.c /app/odbc_helper.c -lodbc -ldl -I/opt/microsoft/msodbcsql17/include" + docker run --rm -v $(CURDIR)/app:/app sql_ae.alpine.build bash -c "gcc -g -fshort-wchar -fPIC -o /app/cksp.so -shared /app/cksp.c -I/opt/microsoft/msodbcsql18/include; gcc -g -o /app/odbc_app -fshort-wchar /app/odbc_app.c /app/odbc_helper.c -lodbc -ldl -I/opt/microsoft/msodbcsql18/include" sudo chown $(USER):$(GROUP) app/odbc_app $(TOP)/scripts/appbuilder Dockerfile cp app/odbc_app $(APPDIR)/odbc_app @@ -35,8 +35,8 @@ app: ls -l $(APPDIR)/odbc_app $(APPDIR)/cksp.so; date run-host: - gcc -g -fshort-wchar -fPIC -o cksp.so -shared app/cksp.c -I/opt/microsoft/msodbcsql17/include - gcc -g -o odbc_app -fshort-wchar app/odbc_app.c app/odbc_helper.c -lodbc -ldl -I/opt/microsoft/msodbcsql17/include + gcc -g -fshort-wchar -fPIC -o cksp.so -shared app/cksp.c -I/opt/microsoft/msodbcsql18/include + gcc -g -o odbc_app -fshort-wchar app/odbc_app.c app/odbc_helper.c -lodbc -ldl -I/opt/microsoft/msodbcsql18/include ./odbc_app msi default clean: diff --git a/solutions/sql_ae/app-dockerfile b/solutions/sql_ae/app-dockerfile index 2eb478cfe..f419e66da 100644 --- a/solutions/sql_ae/app-dockerfile +++ b/solutions/sql_ae/app-dockerfile @@ -5,6 +5,6 @@ RUN apk add --no-cache unixodbc-dev curl build-base bash WORKDIR /app/msodbcinstall -RUN curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.6.1.1-1_amd64.apk && apk add --allow-untrusted $(ls) +RUN curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_18.3.3.1-1_amd64.apk && apk add --allow-untrusted $(ls) diff --git a/solutions/sql_ae/app/odbc_app.c b/solutions/sql_ae/app/odbc_app.c index e1dc81ad4..8a0b3c8d0 100644 --- a/solutions/sql_ae/app/odbc_app.c +++ b/solutions/sql_ae/app/odbc_app.c @@ -52,9 +52,10 @@ int main(int argc, char** argv) CONNSTR_MAX_LEN, "Server=%s;Database=%s;" "Authentication=ActiveDirectoryMsi;" - "Driver={ODBC Driver 17 for SQL Server};" + "Driver={ODBC Driver 18 for SQL Server};" "ColumnEncryption=SGX-AAS,%s/attest/" - "SgxEnclave?api-version=2018-09-01-preview", + "SgxEnclave?api-version=2018-09-01-preview" + "Encrypt=Yes;TrustServerCertificate=Yes", db_server, db_name, maa_url) >= CONNSTR_MAX_LEN) @@ -70,10 +71,11 @@ int main(int argc, char** argv) CONNSTR_MAX_LEN, "Server=%s;Database=%s;" "Authentication=ActiveDirectoryMsi;" - "UID=%s;" - "Driver={ODBC Driver 17 for SQL Server};" + "User Id=%s;" + "Driver={ODBC Driver 18 for SQL Server};" "ColumnEncryption=SGX-AAS,%s/attest/" - "SgxEnclave?api-version=2018-09-01-preview", + "SgxEnclave?api-version=2018-09-01-preview" + "Encrypt=Yes;TrustServerCertificate=Yes", db_server, db_name, db_uid, @@ -102,9 +104,10 @@ int main(int argc, char** argv) CONNSTR_MAX_LEN, "Server=%s;Database=%s;" "UID=%s;PWD=%s;" - "Driver={ODBC Driver 17 for SQL Server};" + "Driver={ODBC Driver 18 for SQL Server};" "ColumnEncryption=SGX-AAS,%s/attest/" - "SgxEnclave?api-version=2018-09-01-preview", + "SgxEnclave?api-version=2018-09-01-preview" + "Encrypt=Yes;TrustServerCertificate=Yes", db_server, db_name, db_uid,