Merge remote-tracking branch 'upstream/master' into feat/fileter_set

* upstream/master: (68 commits)
  fix typos (apache#14950)
  docs: fix custom oauth config (apache#14997)
  fix: apply template_params on external_metadata (apache#14996)
  fix: toggle fullscreen on the dashboard (apache#14979)
  feat(native-filters): add markers and number formatter + simple tests (apache#14981)
  fix(native-filters): Fix "undefined" error after editing a filter (apache#14984)
  fix(native-filters): remove implied fetch predicate (apache#14982)
  fix(native-filters): update cascaded filter state on change (apache#14980)
  fix(filter box): replace freeform where clause with ilike (apache#14900)
  refactor: Convert TableElement.jsx component from class to functional with hooks (apache#14830)
  fix: renamed sqllab filters to _filters (apache#14971)
  feat(native-filters): apply cascading without instant filtering (apache#14966)
  chore: bump superset-ui to 0.17.53 (apache#14968)
  fix(native-filters): cascading filters not rendering in tab (apache#14964)
  feat: add type_generic and is_dttm to table metadata (apache#14863)
  additional safeguard (apache#14953)
  feat: Adding FORCE_SSL as feature flag in config.py (apache#14934)
  feat(dashboard/native-filters): Hide filters out of scope of current tab (apache#14933)
  fix: time parser truncate to first day of year/month (apache#14945)
  fix: is_temporal should overwrite is_dttm (apache#14894)
  ...

.pylintrc

@@ -81,7 +81,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=long-builtin,dict-view-method,intern-builtin,suppressed-message,no-absolute-import,unpacking-in-except,apply-builtin,delslice-method,indexing-exception,old-raise-syntax,print-statement,cmp-builtin,reduce-builtin,useless-suppression,coerce-method,input-builtin,cmp-method,raw_input-builtin,nonzero-method,backtick,basestring-builtin,setslice-method,reload-builtin,oct-method,map-builtin-not-iterating,execfile-builtin,old-octal-literal,zip-builtin-not-iterating,buffer-builtin,getslice-method,metaclass-assignment,xrange-builtin,long-suffix,round-builtin,range-builtin-not-iterating,next-method-called,parameter-unpacking,unicode-builtin,unichr-builtin,import-star-module-level,raising-string,filter-builtin-not-iterating,using-cmp-argument,coerce-builtin,file-builtin,old-division,hex-method,missing-docstring,too-many-lines,ungrouped-imports,import-outside-toplevel,raise-missing-from,super-with-arguments,bad-option-value,too-few-public-methods
+disable=long-builtin,dict-view-method,intern-builtin,suppressed-message,no-absolute-import,unpacking-in-except,apply-builtin,delslice-method,indexing-exception,old-raise-syntax,print-statement,cmp-builtin,reduce-builtin,useless-suppression,coerce-method,input-builtin,cmp-method,raw_input-builtin,nonzero-method,backtick,basestring-builtin,setslice-method,reload-builtin,oct-method,map-builtin-not-iterating,execfile-builtin,old-octal-literal,zip-builtin-not-iterating,buffer-builtin,getslice-method,metaclass-assignment,xrange-builtin,long-suffix,round-builtin,range-builtin-not-iterating,next-method-called,parameter-unpacking,unicode-builtin,unichr-builtin,import-star-module-level,raising-string,filter-builtin-not-iterating,using-cmp-argument,coerce-builtin,file-builtin,old-division,hex-method,missing-docstring,too-many-lines,ungrouped-imports,import-outside-toplevel,raise-missing-from,super-with-arguments,bad-option-value,too-few-public-methods,too-many-locals
 
 
 [REPORTS]

Dockerfile

@@ -84,8 +84,8 @@ ENV LANG=C.UTF-8 \
     SUPERSET_HOME="/app/superset_home" \
     SUPERSET_PORT=8088
 
-RUN useradd --user-group --no-create-home --no-log-init --shell /bin/bash superset \
-        && mkdir -p ${SUPERSET_HOME} ${PYTHONPATH} \
+RUN mkdir -p ${PYTHONPATH} \
+        && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
         && apt-get update -y \
         && apt-get install -y --no-install-recommends \
             build-essential \

docker-compose.yml

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 x-superset-image: &superset-image apache/superset:latest-dev
+x-superset-user: &superset-user root
 x-superset-depends-on: &superset-depends-on
   - db
   - redis
@@ -55,7 +56,7 @@ services:
     restart: unless-stopped
     ports:
       - 8088:8088
-    user: "root"
+    user: *superset-user
     depends_on: *superset-depends-on
     volumes: *superset-volumes
     environment:
@@ -95,7 +96,7 @@ services:
     command: ["/app/docker/docker-init.sh"]
     env_file: docker/.env
     depends_on: *superset-depends-on
-    user: "root"
+    user: *superset-user
     volumes: *superset-volumes
     environment:
       CYPRESS_CONFIG: "${CYPRESS_CONFIG}"
@@ -115,7 +116,7 @@ services:
     env_file: docker/.env
     restart: unless-stopped
     depends_on: *superset-depends-on
-    user: "root"
+    user: *superset-user
     volumes: *superset-volumes
 
   superset-worker-beat:
@@ -125,7 +126,7 @@ services:
     env_file: docker/.env
     restart: unless-stopped
     depends_on: *superset-depends-on
-    user: "root"
+    user: *superset-user
     volumes: *superset-volumes
 
   superset-tests-worker:
@@ -141,7 +142,7 @@ services:
       REDIS_HOST: localhost
     network_mode: host
     depends_on: *superset-depends-on
-    user: "root"
+    user: *superset-user
     volumes: *superset-volumes
 
 volumes:

docker/docker-bootstrap.sh

@@ -41,7 +41,7 @@ if [[ "${1}" == "worker" ]]; then
   celery worker --app=superset.tasks.celery_app:app -Ofair -l INFO
 elif [[ "${1}" == "beat" ]]; then
   echo "Starting Celery beat..."
-  celery beat --app=superset.tasks.celery_app:app --pidfile /tmp/celerybeat.pid -l INFO
+  celery beat --app=superset.tasks.celery_app:app --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
 elif [[ "${1}" == "app" ]]; then
   echo "Starting web app..."
   flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0

docs/installation.rst

@@ -433,7 +433,7 @@ For other strategies, check the `superset/tasks/cache.py` file.
 Caching Thumbnails
 ------------------
 
-This is an optional feature that can be turned on by activating it's feature flag on config:
+This is an optional feature that can be turned on by activating its feature flag on config:
 
 .. code-block:: python
 
@@ -972,7 +972,7 @@ environment variable: ::
 Event Logging
 -------------
 
-Superset by default logs special action event on it's database. These log can be accessed on the UI navigating to
+Superset by default logs special action event on its database. These logs can be accessed on the UI navigating to
 "Security" -> "Action Log". You can freely customize these logs by implementing your own event log class.
 
 Example of a simple JSON to Stdout class::
@@ -1256,6 +1256,38 @@ in this dictionary are made available for users to use in their SQL.
         'my_crazy_macro': lambda x: x*2,
     }
 
+Default values for jinja templates can be specified via ``Parameters`` menu in the SQL Lab user interface.
+In the UI you can assign a set of parameters as JSON
+
+.. code-block:: JSON
+    {
+        "my_table": "foo"
+    }
+
+The parameters become available in your SQL (example:SELECT * FROM {{ my_table }} ) by using Jinja templating syntax.
+SQL Lab template parameters are stored with the dataset as TEMPLATE PARAMETERS.
+
+There is a special ``_filters`` parameter which can be used to test filters used in the jinja template.
+
+.. code-block:: JSON
+    {
+        "_filters": [ {
+            "col": "action_type",
+            "op": "IN",
+            "val": ["sell", "buy"]
+            } ]
+    }
+
+.. code-block:: python
+    SELECT action, count(*) as times
+            FROM logs
+            WHERE
+                action in ({{ "'" + "','".join(filter_values('action_type')) + "'" }})
+            GROUP BY action
+
+Note ``_filters`` is not stored with the dataset. It's only used within the SQL Lab UI.
+
+
 Besides default Jinja templating, SQL lab also supports self-defined template
 processor by setting the ``CUSTOM_TEMPLATE_PROCESSORS`` in your superset configuration.
 The values in this dictionary overwrite the default Jinja template processors of the
@@ -1326,7 +1358,7 @@ The available validators and names can be found in `sql_validators/`.
 **Scheduling queries**
 
 You can optionally allow your users to schedule queries directly in SQL Lab.
-This is done by addding extra metadata to saved queries, which are then picked
+This is done by adding extra metadata to saved queries, which are then picked
 up by an external scheduled (like [Apache Airflow](https://airflow.apache.org/)).
 
 To allow scheduled queries, add the following to your `config.py`:

docs/src/pages/docs/Connecting to Databases/mysql.mdx

@@ -8,16 +8,22 @@ version: 1
 
 ## MySQL
 
-The recommended connector library for MySQL is [mysql-connector-python](https://pypi.org/project/mysql-connector-python/).
+The recommended connector library for MySQL is `[mysqlclient](https://pypi.org/project/mysqlclient/)`.
 
 Here's the connection string:
 
 ```
-mysql+mysqlconnector://{username}:{password}@{host}/{database}
+mysql://{username}:{password}@{host}/{database}
 ```
 
 Host:
 - For Localhost or Docker running Linux: `localhost` or `127.0.0.1`
 - For On Prem: IP address or Host name
 - For Docker running in OSX: `docker.for.mac.host.internal`
 Port: `3306` by default
+
+One problem with `mysqlclient` is that it will fail to connect to newer MySQL databases using `caching_sha2_password` for authentication, since the plugin is not included in the client. In this case, you should use `[mysql-connector-python](https://pypi.org/project/mysql-connector-python/)` instead:
+
+```
+mysql+mysqlconnector://{username}:{password}@{host}/{database}
+```

docs/src/pages/docs/installation/configuring.mdx

@@ -134,7 +134,7 @@ OAUTH_PROVIDERS = [
             'access_token_headers':{    # Additional headers for calls to access_token_url
                 'Authorization': 'Basic Base64EncodedClientIdAndSecret'
             },
-            'base_url':'https://myAuthorizationServer/oauth2AuthorizationServer/',
+            'api_base_url':'https://myAuthorizationServer/oauth2AuthorizationServer/',
             'access_token_url':'https://myAuthorizationServer/oauth2AuthorizationServer/token',
             'authorize_url':'https://myAuthorizationServer/oauth2AuthorizationServer/authorize'
         }

helm/superset/Chart.yaml

@@ -22,7 +22,7 @@ maintainers:
   - name: craig-rueda
     email: [email protected]
     url: https://github.com/craig-rueda
-version: 0.1.2
+version: 0.1.3
 dependencies:
 - name: postgresql
   version: 10.2.0

helm/superset/templates/deployment-beat.yaml

@@ -96,6 +96,10 @@ spec:
       tolerations:
 {{ toYaml . | indent 8 }}
     {{- end }}
+{{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.imagePullSecrets | indent 8 }}
+      {{- end }}
       volumes:
         - name: superset-config
           secret:

helm/superset/templates/deployment-worker.yaml

@@ -94,6 +94,10 @@ spec:
       tolerations:
 {{ toYaml . | indent 8 }}
     {{- end }}
+{{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.imagePullSecrets | indent 8 }}
+      {{- end }}
       volumes:
         - name: superset-config
           secret:

helm/superset/templates/deployment.yaml

@@ -106,6 +106,11 @@ spec:
       tolerations:
 {{ toYaml . | indent 8 }}
     {{- end }}
+{{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.imagePullSecrets | indent 8 }}
+      {{- end }}
+
       volumes:
         - name: superset-config
           secret:

helm/superset/templates/init-job.yaml

@@ -59,6 +59,10 @@ spec:
         command: {{  tpl (toJson .Values.init.command) . }}
         resources:
 {{ toYaml .Values.init.resources | indent 10 }}
+{{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.imagePullSecrets | indent 8 }}
+      {{- end }}
       volumes:
         - name: superset-config
           secret:

helm/superset/values.yaml

@@ -106,6 +106,9 @@ image:
   tag: latest
   pullPolicy: IfNotPresent
 
+imagePullSecrets: []
+
+
 service:
   type: ClusterIP
   port: 8088

requirements/base.txt

@@ -27,7 +27,7 @@ backoff==1.10.0
     # via apache-superset
 billiard==3.6.3.0
     # via celery
-bleach==3.2.1
+bleach==3.3.0
     # via apache-superset
 brotli==1.0.9
     # via flask-compress
@@ -56,7 +56,7 @@ cron-descriptor==1.2.24
     # via apache-superset
 croniter==0.3.36
     # via apache-superset
-cryptography==3.2.1
+cryptography==3.3.2
     # via apache-superset
 decorator==4.4.2
     # via retry

setup.py

@@ -73,7 +73,7 @@ def get_git_sha():
         "contextlib2",
         "croniter>=0.3.28",
         "cron-descriptor",
-        "cryptography>=3.2.1",
+        "cryptography>=3.3.2",
         "flask>=1.1.0, <2.0.0",
         "flask-appbuilder>=3.3.0, <4.0.0",
         "flask-caching",

superset-frontend/cypress-base/cypress/integration/dashboard/filter.test.ts

@@ -70,8 +70,8 @@ describe('Dashboard filter', () => {
           }
           expect(requestFilter).deep.eq({
             col: 'region',
-            op: '==',
-            val: 'South Asia',
+            op: 'IN',
+            val: ['South Asia'],
           });
         });
       });

superset-frontend/cypress-base/cypress/integration/dashboard/tabs.test.ts

@@ -121,8 +121,8 @@ describe('Dashboard tabs', () => {
         const requestParams = JSON.parse(requestBody.form_data as string);
         expect(requestParams.extra_filters[0]).deep.eq({
           col: 'region',
-          op: '==',
-          val: 'South Asia',
+          op: 'IN',
+          val: ['South Asia'],
         });
       });
     });
@@ -136,8 +136,8 @@ describe('Dashboard tabs', () => {
       const requestParams = JSON.parse(requestBody.form_data as string);
       expect(requestParams.extra_filters[0]).deep.eq({
         col: 'region',
-        op: '==',
-        val: 'South Asia',
+        op: 'IN',
+        val: ['South Asia'],
       });
       expect(requestParams.viz_type).eq(LINE_CHART.viz);
     });
@@ -150,8 +150,8 @@ describe('Dashboard tabs', () => {
     cy.wait('@v1ChartData').then(({ request }) => {
       expect(request.body.queries[0].filters[0]).deep.eq({
         col: 'region',
-        op: '==',
-        val: 'South Asia',
+        op: 'IN',
+        val: ['South Asia'],
       });
     });
 

superset-frontend/cypress-base/cypress/integration/dashboard_list/filter.test.ts

@@ -61,7 +61,6 @@ describe('dashboard filters card view', () => {
     cy.get('.Select__menu').contains('Published').click({ timeout: 5000 });
     cy.get('[data-test="styled-card"]').should('have.length', 2);
     cy.get('[data-test="styled-card"]')
-      .first()
       .contains('USA Births Names')
       .should('be.visible');
     cy.get('.Select__control').eq(1).click();
@@ -107,13 +106,12 @@ describe('dashboard filters list view', () => {
     cy.get('[data-test="table-row"]').should('not.exist');
   });
 
-  xit('should filter by published correctly', () => {
+  it('should filter by published correctly', () => {
     // filter by published
     cy.get('.Select__control').eq(2).click();
     cy.get('.Select__menu').contains('Published').click();
     cy.get('[data-test="table-row"]').should('have.length', 2);
     cy.get('[data-test="table-row"]')
-      .first()
       .contains('USA Births Names')
       .should('be.visible');
     cy.get('.Select__control').eq(2).click();

superset-frontend/cypress-base/cypress/integration/explore/visualizations/line.test.ts

@@ -29,7 +29,7 @@ describe('Visualization > Line', () => {
   it('should show validator error when no metric', () => {
     const formData = { ...LINE_CHART_DEFAULTS, metrics: [] };
     cy.visitChartByParams(JSON.stringify(formData));
-    cy.get('.ant-alert-warning').contains(`"Metrics" cannot be empty`);
+    cy.get('.ant-alert-warning').contains(`Metrics: cannot be empty`);
   });
 
   it('should preload mathjs', () => {
@@ -43,7 +43,7 @@ describe('Visualization > Line', () => {
   it('should not show validator error when metric added', () => {
     const formData = { ...LINE_CHART_DEFAULTS, metrics: [] };
     cy.visitChartByParams(JSON.stringify(formData));
-    cy.get('.ant-alert-warning').contains(`"Metrics" cannot be empty`);
+    cy.get('.ant-alert-warning').contains(`Metrics: cannot be empty`);
     cy.get('.text-danger').contains('Metrics');
 
     cy.get('[data-test=metrics]')
@@ -62,6 +62,8 @@ describe('Visualization > Line', () => {
   });
 
   it('should allow negative values in Y bounds', () => {
+    const formData = { ...LINE_CHART_DEFAULTS, metrics: [NUM_METRIC] };
+    cy.visitChartByParams(JSON.stringify(formData));
     cy.get('#controlSections-tab-display').click();
     cy.get('span').contains('Y Axis Bounds').scrollIntoView();
     cy.get('input[placeholder="Min"]').type('-0.1', { delay: 100 });