Support version 2 of Flamenco Manager variables

A restructuring of the Manager variables was necessary to allow different values for users and workers. Until now both had the same paths, which is fine when both are located in the same studio. However, when deploying workers on a cloud infrastructure (Azure, AWS, etc.) this becomes cumbersome. When a Manager does not declare a `settings_version`, version 1 is assumed. This commit introduces version 2. The naming 'path replacement variable' was confusing people. Now each variable has a mandatory property 'direction' that has one of the following values: - `"one"`: unidirectional variable. `${VARIABLENAME}` is replaced by the variable value for the appropriate platform. This used to be the 'variable'. - `"two"`: bidirection variable. As described above, but also the value is replaced by `${VARIABLENAME}` by the Blender Cloud add-on when sending a job to Flamenco Server. This used to be the 'path replacement variable'. Each variable now has a setting 'audience', which can be either 'workers' or 'users' to be applied only to those, or 'all' when the value is the same for both.
armadillica · Apr 18, 2019 · 6fc2d65 · 6fc2d65
1 parent aaeb6ee
commit 6fc2d65
Show file tree

Hide file tree

Showing 13 changed files with 344 additions and 52 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,11 @@
 Flamenco Server Changelog
 =========================
 
+## Version 2.3 (in development)
+
+- Support for Flamenco Manager settings version 2; this is introduced in Flamenco Manager 2.5.
+
+
 ## Version 2.2 (released 2019-03-25)
 
 - Requires Flamenco Worker 2.3 or newer.

diff --git a/flamenco/blender_cloud_addon.py b/flamenco/blender_cloud_addon.py
@@ -0,0 +1,27 @@
+import typing
+
+import flask
+import werkzeug.exceptions as wz_exceptions
+
+
+def requested_by_version() -> typing.Optional[typing.Tuple[int, int, int]]:
+    """Determine the version of the Blender Cloud add-on.
+
+    The version of the Blender Cloud add-on performing the current request is
+    returned as (major, minor, micro) tuple. If the current request did not
+    come from the Blender Cloud add-on (as recognised by the
+    'Blender-Cloud-Addon' HTTP header), return None.
+    """
+    addon_version = flask.request.headers.get('Blender-Cloud-Addon')
+    if not addon_version:
+        return None
+
+    try:
+        parts = tuple(int(part) for part in addon_version.split('.'))
+    except ValueError:
+        raise wz_exceptions.BadRequest('Invalid Blender-Cloud-Addon header')
+
+    if 2 <= len(parts) < 4:
+        return (parts + (0, 0))[:3]
+
+    raise wz_exceptions.BadRequest('Invalid Blender-Cloud-Addon header')
diff --git a/flamenco/eve_settings.py b/flamenco/eve_settings.py
@@ -101,7 +101,12 @@
         }
     },
 
-    # Received from the manager itself at startup.
+    # Received from the manager itself at startup in {'_meta': {'version': N}}.
+    'settings_version': {
+        'type': 'integer',
+        'default': 1,
+    },
+    # 'variables' is used both versions 1 and 2, 'path_replacement' only version 1.
     'variables': {
         'type': 'dict',
         'allow_unknown': True,

diff --git a/flamenco/jobs/routes.py b/flamenco/jobs/routes.py
@@ -157,7 +157,7 @@ def view_job(project, flamenco_props, job_id):
         'flamenco/jobs/view_job_embed.html',
         job=job,
         user_name=user_name,
-        manager=manager,
+        manager=manager.to_dict(),
         project=project,
         flamenco_props=flamenco_props.to_dict(),
         flamenco_context=request.args.get('context'),

diff --git a/flamenco/managers/api.py b/flamenco/managers/api.py
@@ -92,28 +92,35 @@ def handle_notification(manager_id: str, notification: dict):
     if not notification:
         raise wz_exceptions.BadRequest('no JSON payload received')
 
+    settings_version = notification.get('_meta', {}).get('version', 1)
+    updates_unset = {}
     try:
-        updates = {
+        updates_set = {
             '_updated': datetime.datetime.utcnow(),
             '_etag': uuid.uuid4().hex,
             'url': notification['manager_url'],
+            'settings_version': settings_version,
             'variables': notification['variables'],
-            'path_replacement': notification['path_replacement'],
             'stats.nr_of_workers': notification['nr_of_workers'],
         }
+        if settings_version <= 1:
+            updates_set['path_replacement'] = notification['path_replacement']
+        else:
+            updates_unset['path_replacement'] = True
     except KeyError as ex:
         raise wz_exceptions.BadRequest(f'Missing key {ex}')
 
     try:
-        updates['worker_task_types'] = notification['worker_task_types']
+        updates_set['worker_task_types'] = notification['worker_task_types']
     except KeyError:
         pass
 
     mngr_coll = current_flamenco.db('managers')
-    update_res = mngr_coll.update_one(
-        {'_id': manager_id},
-        {'$set': updates}
-    )
+    updates = {'$set': updates_set}
+    if updates_unset:
+        updates['$unset'] = updates_unset
+
+    update_res = mngr_coll.update_one({'_id': manager_id}, updates)
     if update_res.matched_count != 1:
         log.warning('Updating manager %s matched %i documents.',
                     manager_id, update_res.matched_count)

diff --git a/flamenco/managers/eve_hooks.py b/flamenco/managers/eve_hooks.py
@@ -1,13 +1,16 @@
+import collections
 import logging
+import re
+import typing
 
 import werkzeug.exceptions as wz_exceptions
 
 from pillar.auth import current_user
 
-from flamenco import current_flamenco
-import flamenco.auth
+from flamenco import current_flamenco, blender_cloud_addon
 
 log = logging.getLogger(__name__)
+field_name_escape_replace = re.compile('[.$]')
 
 
 def check_manager_permissions(mngr_doc):
@@ -72,10 +75,64 @@ def pre_get_flamenco_managers(request, lookup):
     log.debug('Filtering on %s', lookup)
 
 
+def rewrite_manager_settings(doc: dict):
+    """Update the Manager's variables to be compatible with the Blender Cloud add-on.
+
+    The Blender Cloud Add-on only implemented versioning of Manager settings in
+    1.13, so if an older version requests the Manager just transform it to a version
+    the add-on understands.
+    """
+
+    # Make sure the version is always explicit.
+    doc.setdefault('settings_version', 1)
+
+    addon_version = blender_cloud_addon.requested_by_version()
+    if not addon_version or addon_version >= (1, 12, 2):
+        return
+
+    # Downgrade settings for this old add-on version.
+    # Since it's the Blender Cloud add-on, we're targeting the 'users' audience.
+    audiences = {'', 'all', 'users'}
+
+    # variable name -> platform -> value
+    oneway = collections.defaultdict(lambda: collections.defaultdict(dict))
+    twoway = collections.defaultdict(lambda: collections.defaultdict(dict))
+    target_maps = {
+        'oneway': oneway,
+        'twoway': twoway,
+    }
+
+    for name, variable in doc.get('variables', {}).items():
+        direction = variable.get('direction', 'oneway')
+        target_map = target_maps.get(direction, oneway)
+
+        for value in variable['values']:
+            if value['audience'] not in audiences:
+                continue
+
+            if value.get('platform'):
+                target_map[name][value['platform']] = value['value']
+
+            for platform in value.get('platforms', []):
+                target_map[name][platform] = value['value']
+
+    doc['variables'] = oneway
+    doc['path_replacement'] = twoway
+    doc['settings_version'] = 1
+
+
+def rewrite_managers_settings(response: dict):
+    for manager in response['_items']:
+        manager.setdefault('settings_version', 1)
+        rewrite_manager_settings(manager)
+
+
 def setup_app(app):
     app.on_pre_GET_flamenco_managers += pre_get_flamenco_managers
     app.on_fetched_item_flamenco_managers += check_manager_permissions
+    app.on_fetched_item_flamenco_managers += rewrite_manager_settings
     app.on_fetched_resource_flamenco_managers += check_manager_resource_permissions
+    app.on_fetched_resource_flamenco_managers += rewrite_managers_settings
     app.on_insert_flamenco_managers += check_manager_permissions_create
     app.on_update_flamenco_managers += check_manager_permissions_modify
     app.on_replace_flamenco_managers += check_manager_permissions_modify

diff --git a/flamenco/managers/routes.py b/flamenco/managers/routes.py
@@ -95,7 +95,7 @@ def view_embed(manager_id: str):
     csrf = flask_wtf.csrf.generate_csrf()
 
     return render_template('flamenco/managers/view_manager_embed.html',
-                           manager=manager,
+                           manager=manager.to_dict(),
                            can_edit=can_edit,
                            available_projects=available_projects,
                            linked_projects=linked_projects,

diff --git a/src/styles/_app_base.sass b/src/styles/_app_base.sass
@@ -240,6 +240,14 @@ nav.sidebar
 				.table-cell
 					&.variable-key
 						white-space: nowrap
+					&.variable-direction
+						cursor: help
+						font-size: large !important
+						vertical-align: top !important
+			&.odd
+				background-color: lighten($color-background, 7%)
+			&.even
+				background-color: lighten($color-background, 8%)
 
 	.item-id
 		padding: 8px

diff --git a/src/templates/flamenco/jobs/view_job_embed.pug b/src/templates/flamenco/jobs/view_job_embed.pug
@@ -157,16 +157,33 @@ script.
 		let osName = bowser.getParser(window.navigator.userAgent).getOSName();
 		if (osName != 'Windows' && osName != 'macOS') osName = 'Linux'
 
-		let mngrVariables = {{ manager.variables.to_dict() | tojson }};
-		let mngrPathReplacement = {{ manager.path_replacement.to_dict() | tojson }};
+		let platformVars = {};
+		let osname = osName.toLowerCase();
+	{% if manager.settings_version < 2 %}
+		let mngrVariables = {{ manager.variables | tojson }};
+		let mngrPathReplacement = {{ manager.path_replacement | tojson }};
 
 		// Convert the per-OS variables into a flat OS-specific structure.
-		var platformVars = {};
-		let osname = osName.toLowerCase();
 		for (varname in mngrVariables)
 			platformVars[varname] = mngrVariables[varname][osname];
 		for (varname in mngrPathReplacement)
 			platformVars[varname] = mngrPathReplacement[varname][osname];
+	{% elif manager.settings_version == 2 %}
+		let mngrVariables = {{ manager.variables | tojson }};
+		//- A variable's 'audience' is either "all", "workers", or "users",
+		//-  and not all are interesting to the user.
+		let userAudience = {all: true, user: true};
+
+		// Convert the per-OS variables into a flat OS-specific structure.
+		for (varName in mngrVariables) {
+			for (valueDef of mngrVariables[varName].values) {
+				if (valueDef.platform.toLowerCase() != osname) continue;
+				if (userAudience[valueDef.audience.toLowerCase()] !== true) continue;
+
+				platformVars[varName] = valueDef.value;
+			}
+		}
+	{% endif %}
 
 		// Some settings need some special treatment.
 		// Mapping from setting key to function that transforms the setting value.

diff --git a/src/templates/flamenco/managers/_variables.pug b/src/templates/flamenco/managers/_variables.pug
@@ -0,0 +1,59 @@
+| {% macro variables_view(manager) -%}
+| {% if manager.settings_version < 2 %}
+//- ---------- SETTINGS version 1 --------------
+| {% if manager.variables %}
+h4 Variables
+| {% for var_key, var_values in manager.variables.items()  %}
+.table.item-properties
+    .table-body(title='Unable to edit, set by Manager')
+        | {% for v in var_values %}
+        .table-row.properties-variables
+            .table-cell.variable-name {% if loop.first %}{{ var_key }}{% endif %}
+            .table-cell.variable-key {{ v }}
+            .table-cell.variable-value {{ var_values[v] }}
+        | {% endfor %}
+| {% endfor %}
+| {% endif %}
+
+| {% if manager.path_replacement %}
+h4 Path Replacement Variables
+| {% for pathrep_key, pathrep_values in manager.path_replacement.items()  %}
+.table.item-properties
+    .table-body(title='Unable to edit, set by Manager')
+        | {% for v in pathrep_values %}
+        .table-row.properties-path-replacement
+            .table-cell.variable-name {% if loop.first %}{{ pathrep_key }}{% endif %}
+            .table-cell.variable-key {{ v }}
+            .table-cell.variable-value {{ pathrep_values[v] }}
+        | {% endfor %}
+| {% endfor %}
+| {% endif %}
+
+//- ---------- SETTINGS version 2 --------------
+| {% elif manager.settings_version == 2 %}
+| {% if manager.variables %}
+h4 Variables
+
+.table.item-properties
+    .table-body(title='Unable to edit, set by Manager')
+        | {% for var_key, var_definition in manager.variables.items()  %}
+        | {% set evenodd = loop.cycle('odd', 'even') %}
+        | {% for v in var_definition['values'] %}
+        .table-row.properties-variables(class="{{ evenodd }}")
+            .table-cell.variable-name {% if loop.first %}{{ var_key }}{% endif %}
+            .table-cell.variable-direction(title="{% if var_definition.direction == 'twoway' %}Two-way, values are replaced by variables and vice versa{%else%}One-way, variables are replaced with values{%endif%}")
+                | {% if loop.first %}{% if var_definition.direction == "twoway" %}⇄{% else %}→{% endif %}{% endif %}
+            .table-cell.variable-value {{ v.value }}
+            .table-cell.variable-platform {{ v.platform }} {{ v.platforms|join(' ') }}
+            .table-cell.variable-audience {{ v.audience }}
+        | {% endfor %}
+        | {% endfor %}
+
+
+| {% endif %}
+
+| {% else %}
+h4 Unsupported version
+p This manager has settings version {{ manager.settings_version }}, which we don't support.
+| {% endif %}
+| {%- endmacro %}
diff --git a/src/templates/flamenco/managers/view_manager_embed.pug b/src/templates/flamenco/managers/view_manager_embed.pug
@@ -1,3 +1,4 @@
+| {% from "flamenco/managers/_variables.html" import variables_view %}
 .flamenco-box.manager
 	form#item_form(onsubmit="return editManager()")
 		| {% if can_edit %}
@@ -56,33 +57,7 @@
 					| never sent by Manager
 					| {% endif %}
 
-	| {% if manager.variables %}
-	h4 Variables
-	| {% for var_key, var_values in manager.variables.to_dict().items()  %}
-	.table.item-properties
-		.table-body(title='Unable to edit, set by Manager')
-			| {% for v in var_values %}
-			.table-row.properties-variables
-				.table-cell.variable-name {% if loop.first %}{{ var_key }}{% endif %}
-				.table-cell.variable-key {{ v }}
-				.table-cell.variable-value {{ var_values[v] }}
-			| {% endfor %}
-	| {% endfor %}
-	| {% endif %}
-
-	| {% if manager.path_replacement %}
-	h4 Path Replacement Variables
-	| {% for pathrep_key, pathrep_values in manager.path_replacement.to_dict().items()  %}
-	.table.item-properties
-		.table-body(title='Unable to edit, set by Manager')
-			| {% for v in pathrep_values %}
-			.table-row.properties-path-replacement
-				.table-cell.variable-name {% if loop.first %}{{ pathrep_key }}{% endif %}
-				.table-cell.variable-key {{ v }}
-				.table-cell.variable-value {{ pathrep_values[v] }}
-			| {% endfor %}
-	| {% endfor %}
-	| {% endif %}
+	| {{ variables_view(manager) }}
 
 .flamenco-box.manager
 	h4 Projects linked to this Manager
@@ -219,7 +194,7 @@
 			| Debug Info
 		#debug-content-manager.collapse
 			pre.
-				{{ manager.to_dict() | pprint }}
+				{{ manager | pprint }}
 	| {% endif %}
 
 | {% block footer_scripts %}