Skip to content

Commit

Permalink
[Cherry-Pick] Dynamo ML data ingestion Pipeline Extension. (#14749) (#…
Browse files Browse the repository at this point in the history
…14911)

* Dynamo ML data ingestion Pipeline Extension. (#14749)

* commit

* updates

* updates

* Update DynamoCoreWpf.csproj

* Update DynamoCore.csproj

* remove extra whitespaces

* updates

* updates

* new changes

* switch to prod

* add to dynamocore.sln

* Control by feature flag.

* Update DynamoMLDataPipelineExtension.cs

* Update DynamoMLDataPipelineExtension.cs

* Update DynamoMLDataPipeline.csproj

* Separating the extension code from core code and addressing comments.

* Update DynamoMLDataPipeline.csproj

* Update to feature flag and some comments

* Fix Analytics reference error

* Fix failing tests.

* Remove beta keyword

* Make the test as failure

---------

Co-authored-by: reddyashish <[email protected]>
  • Loading branch information
QilongTang and reddyashish authored Jan 30, 2024
1 parent 762914c commit 79a3c97
Show file tree
Hide file tree
Showing 29 changed files with 1,000 additions and 27 deletions.
6 changes: 6 additions & 0 deletions src/Dynamo.All.sln
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WpfVisualizationTests", "..
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DynamoPythonTests", "..\test\Libraries\DynamoPythonTests\DynamoPythonTests.csproj", "{773988FE-EDF6-45CB-A63F-482955EB3553}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DynamoMLDataPipeline", "DynamoMLDataPipeline\DynamoMLDataPipeline.csproj", "{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -524,6 +526,10 @@ Global
{773988FE-EDF6-45CB-A63F-482955EB3553}.Debug|Any CPU.Build.0 = Debug|Any CPU
{773988FE-EDF6-45CB-A63F-482955EB3553}.Release|Any CPU.ActiveCfg = Release|Any CPU
{773988FE-EDF6-45CB-A63F-482955EB3553}.Release|Any CPU.Build.0 = Release|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
10 changes: 10 additions & 0 deletions src/DynamoCore.sln
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DSOffice", "Libraries\DSOff
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DSOfficeUtilities", "Libraries\DSOfficeUtilities\DSOfficeUtilities.csproj", "{9B4FDC96-E2F9-4B8F-894A-4294405D50E7}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DynamoMLDataPipeline", "DynamoMLDataPipeline\DynamoMLDataPipeline.csproj", "{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|NET60_Linux = Debug|NET60_Linux
Expand Down Expand Up @@ -275,6 +277,14 @@ Global
{47533B7C-0E1A-44A4-8511-B438645F052A}.Release|NET60_Linux.Build.0 = Release|NET60_Linux
{47533B7C-0E1A-44A4-8511-B438645F052A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{47533B7C-0E1A-44A4-8511-B438645F052A}.Release|Any CPU.Build.0 = Release|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Debug|NET60_Linux.ActiveCfg = Debug|NET60_Linux
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Debug|NET60_Linux.Build.0 = Debug|NET60_Linux
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Release|NET60_Linux.ActiveCfg = Release|NET60_Linux
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Release|NET60_Linux.Build.0 = Release|NET60_Linux
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5DF79F45-5F2C-41C1-BACC-890AE514CDA8}.Release|Any CPU.Build.0 = Release|Any CPU
{C0D6DEE5-5532-4345-9C66-4C00D7FDB8BE}.Debug|NET60_Linux.ActiveCfg = Debug|NET60_Linux
{C0D6DEE5-5532-4345-9C66-4C00D7FDB8BE}.Debug|NET60_Linux.Build.0 = Debug|NET60_Linux
{C0D6DEE5-5532-4345-9C66-4C00D7FDB8BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
Expand Down
24 changes: 12 additions & 12 deletions src/DynamoCore/DynamoCore.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
<PackageReference Include="DynamoVisualProgramming.LibG_229_0_0" Version="3.0.0.3331" GeneratePathProperty="true" />
<PackageReference Include="DynamoVisualProgramming.LibG_230_0_0" Version="3.0.0.3421" GeneratePathProperty="true" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" CopyXML="true" />
<PackageReference Include="RestSharp" Version="108.0.1" CopyXML="true" />
<PackageReference Include="Lucene.Net" Version="4.8.0-beta00016" />
<PackageReference Include="RestSharp" Version="108.0.1" CopyXML="true" />
<PackageReference Include="Lucene.Net" Version="4.8.0-beta00016" />
<PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00016" />
<PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00016" />
<PackageReference Include="DynamoVisualProgramming.Analytics" Version="4.2.0.3534" />
Expand Down Expand Up @@ -122,12 +122,12 @@
<LibG229 Include="$(PkgDynamoVisualProgramming_LibG_229_0_0)\tools\netstandard2.0\$(LibGOsToken)\LibG_229_0_0\*" />
<SampleFiles Include="$(SolutionDir)..\doc\distrib\Samples\**\*.*" />
<NodeHelpMDFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.md" />
<NodeHelpDYNFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.dyn" />
<NodeHelpTXTFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.txt" />
<NodeHelpSATFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.sat" />
<NodeHelpDYNFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.dyn" />
<NodeHelpTXTFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.txt" />
<NodeHelpSATFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.sat" />
<NodeHelpJpgImageFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.jpg" />
<NodeHelpPngImageFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.png" />
<NodeHelpGifImageFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.gif" />
<NodeHelpPngImageFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.png" />
<NodeHelpGifImageFiles Include="$(SolutionDir)..\doc\distrib\NodeHelpFiles\**\*.gif" />
<OpenSourceLicenses Include="$(SolutionDir)..\doc\distrib\Open Source Licenses\**\*.*" />
<LocalizedResources Include="$(SolutionDir)..\extern\Localized\**\*.*" />
</ItemGroup>
Expand All @@ -137,12 +137,12 @@
<Copy SourceFiles="$(SolutionDir)..\doc\distrib\ADPAnalyticsConsent.rtf" DestinationFolder="$(OutputPath)" />
<Copy SourceFiles="$(SolutionDir)..\doc\distrib\MLNodeAutocompleteConsent.rtf" DestinationFolder="$(OutputPath)" />
<Copy SourceFiles="@(NodeHelpMDFiles)" DestinationFolder="$(OutputPath)\en-US\fallback_docs\" />
<Copy SourceFiles="@(NodeHelpDYNFiles)" DestinationFolder="$(OutputPath)\NodeHelpSharedDocs\" />
<Copy SourceFiles="@(NodeHelpTXTFiles)" DestinationFolder="$(OutputPath)\en-US\fallback_docs\" />
<Copy SourceFiles="@(NodeHelpSATFiles)" DestinationFolder="$(OutputPath)\en-US\fallback_docs\" />
<Copy SourceFiles="@(NodeHelpDYNFiles)" DestinationFolder="$(OutputPath)\NodeHelpSharedDocs\" />
<Copy SourceFiles="@(NodeHelpTXTFiles)" DestinationFolder="$(OutputPath)\en-US\fallback_docs\" />
<Copy SourceFiles="@(NodeHelpSATFiles)" DestinationFolder="$(OutputPath)\en-US\fallback_docs\" />
<Copy SourceFiles="@(NodeHelpJpgImageFiles)" DestinationFolder="$(OutputPath)\NodeHelpSharedDocs\" />
<Copy SourceFiles="@(NodeHelpPngImageFiles)" DestinationFolder="$(OutputPath)\NodeHelpSharedDocs\" />
<Copy SourceFiles="@(NodeHelpGifImageFiles)" DestinationFolder="$(OutputPath)\NodeHelpSharedDocs\" />
<Copy SourceFiles="@(NodeHelpPngImageFiles)" DestinationFolder="$(OutputPath)\NodeHelpSharedDocs\" />
<Copy SourceFiles="@(NodeHelpGifImageFiles)" DestinationFolder="$(OutputPath)\NodeHelpSharedDocs\" />
<Copy SourceFiles="@(OpenSourceLicenses)" DestinationFolder="$(OutputPath)Open Source Licenses\" />
<Copy SourceFiles="@(LibGInterface)" DestinationFolder="$(OutputPath)" />
<Copy SourceFiles="@(LibGProtoGeometryDLL)" DestinationFolder="$(OutputPath)" />
Expand Down
2 changes: 1 addition & 1 deletion src/DynamoCore/Extensions/ExtensionLoader.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
Expand Down
1 change: 1 addition & 0 deletions src/DynamoCore/Properties/AssemblyInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
// For workspace package dependency collection
[assembly: InternalsVisibleTo("DynamoPackages")]
[assembly: InternalsVisibleTo("WorkspaceDependencyViewExtension")]
[assembly: InternalsVisibleTo("DynamoMLDataPipeline")]
[assembly: InternalsVisibleTo("PythonNodeModelsWpf")]
[assembly: InternalsVisibleTo("PythonNodeModels")]
[assembly: InternalsVisibleTo("LibraryViewExtensionWebView2")]
Expand Down
7 changes: 6 additions & 1 deletion src/DynamoCoreWpf/DynamoCoreWpf.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -1557,6 +1557,11 @@
<Name>DynamoPackages</Name>
<Private>False</Private>
</ProjectReference>
<ProjectReference Include="..\DynamoMLDataPipeline\DynamoMLDataPipeline.csproj">
<Project>{F2FCFD49-84BC-4C0E-90C1-5A327D017C4E}</Project>
<Name>DynamoMLDataPipeline</Name>
<Private>False</Private>
</ProjectReference>
<ProjectReference Include="..\DynamoUtilities\DynamoUtilities.csproj">
<Project>{b5f435cb-0d8a-40b1-a4f7-5ecb3ce792a9}</Project>
<Name>DynamoUtilities</Name>
Expand Down Expand Up @@ -1779,4 +1784,4 @@
<Target Name="AfterBuildOps" AfterTargets="Build">
<MakeDir Directories="$(OutputPath)\viewExtensions\" />
</Target>
</Project>
</Project>
20 changes: 18 additions & 2 deletions src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
using Dynamo.Wpf.ViewModels.Core.Converters;
using Dynamo.Wpf.ViewModels.FileTrust;
using Dynamo.Wpf.ViewModels.Watch3D;
using DynamoMLDataPipeline;
using DynamoUtilities;
using ICSharpCode.AvalonEdit;
using PythonNodeModels;
Expand Down Expand Up @@ -85,6 +86,8 @@ public partial class DynamoViewModel : ViewModelBase, IDynamoViewModel
/// </summary>
internal Dictionary<string, ViewExtensionDisplayMode> NodeWindowsState { get; set; } = new Dictionary<string, ViewExtensionDisplayMode>();

internal DynamoMLDataPipelineExtension MLDataPipelineExtension { get; set; }

/// <summary>
/// Collection of Right SideBar tab items: view extensions and docked windows.
/// </summary>
Expand Down Expand Up @@ -191,6 +194,17 @@ public WorkspaceModel CurrentSpace
get { return model.CurrentWorkspace; }
}

/// <summary>
/// Controls if the the ML data ingestion pipeline is beta from feature flag
/// </summary>
internal bool IsMLDataIngestionPipelineinBeta
{
get
{
return DynamoModel.FeatureFlags?.CheckFeatureFlag("IsMLDataIngestionPipelineinBeta", false) ?? false;
}
}

/// <summary>
/// Count of unresolved issues on the linter manager.
/// This is used for binding in the NotificationsControl
Expand Down Expand Up @@ -752,6 +766,7 @@ protected DynamoViewModel(StartConfiguration startConfiguration)
}

FileTrustViewModel = new FileTrustWarningViewModel();
MLDataPipelineExtension = model.ExtensionManager.Extensions.OfType<DynamoMLDataPipelineExtension>().FirstOrDefault();
}

/// <summary>
Expand Down Expand Up @@ -2099,14 +2114,15 @@ private void InternalSaveAs(string path, SaveContext saveContext, bool isBackup
{
AddToRecentFiles(path);

if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash && Model.CurrentWorkspace.IsValidForFDX && currentWorkspaceViewModel.Checksum != string.Empty)
if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash && Model.CurrentWorkspace.IsValidForFDX && IsMLDataIngestionPipelineinBeta && currentWorkspaceViewModel.Checksum != string.Empty)
{
Model.Logger.Log("The Workspace is valid for FDX");
Model.Logger.Log("The Workspace id is : " + currentWorkspaceViewModel.Model.Guid.ToString());
Model.Logger.Log("The Workspace checksum is : " + currentWorkspaceViewModel.Checksum);
Model.Logger.Log("The Workspace has Substantial checksum, so is ready to send to FDX : " + HasSubstantialCheckSum().ToString());
MLDataPipelineExtension.DynamoMLDataPipeline.DataExchange(path);
}
}
}
}
catch (Exception ex)
{
Expand Down
20 changes: 20 additions & 0 deletions src/DynamoMLDataPipeline/App.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<appSettings>
<add key="StagingClientUrl" value="https://developer-stg.api.autodesk.com/exchange"/>
<add key="StagingCollectionID" value="9R09ArUBUEDVRIGQ5OE373_L2C"/>
<add key="ProductionClientUrl" value="https://developer.api.autodesk.com/exchange"/>
<add key="ProductionCollectionID" value="jcQm4Ir2Z8L4ANOmziL1rd_L2C"/>
</appSettings>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8" />
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Runtime.CompilerServices.Unsafe" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>
25 changes: 25 additions & 0 deletions src/DynamoMLDataPipeline/Attribute.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
using Newtonsoft.Json;

namespace DynamoMLDataPipeline
{
// Attributes for the data request object.
class Attribute
{
[JsonProperty("category")]
public string Category { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("value")]
public string Value { get; set; }
[JsonProperty("type")]
public string Type { get; set; }

public Attribute(string name, string value, string category = "application", string type = "String")
{
Category = category;
Name = name;
Value = value;
Type = type;
}
}
}
32 changes: 32 additions & 0 deletions src/DynamoMLDataPipeline/BaseComponent.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
using System.Collections.Generic;
using Newtonsoft.Json;

namespace DynamoMLDataPipeline
{
class BaseComponent : Dictionary<string, Dictionary<string, ObjectInfo>>
{
private string objectId = "autodesk.design:components.base-1.0.0";
public BaseComponent(string name)
{
var objectInfo = new ObjectInfo(name);
var item = new Dictionary<string, ObjectInfo>
{
{ "String", objectInfo }
};
this.Add("objectInfo", item);
}

public string ObjectId { get { return objectId; } }
}

class ObjectInfo
{
[JsonProperty("name")]
public string Name { get; set; }

public ObjectInfo(string name)
{
Name = name;
}
}
}
55 changes: 55 additions & 0 deletions src/DynamoMLDataPipeline/BinaryAsset.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using Newtonsoft.Json;


namespace DynamoMLDataPipeline
{
class BaseBinaryAsset
{
[JsonProperty("id")]
public string Id { get; set; }
}

class UploadedBinaryAsset : BaseBinaryAsset
{
public UploadedBinaryAsset(string guid)
{
Id = guid;
}
}

class BinaryAsset : BaseBinaryAsset
{
public BinaryAsset()
{
Id = Guid.NewGuid().ToString("N").ToUpper();
Parts = 1;
IncludeUploadUrl = true;
Type = "single";
}

[JsonProperty("parts")]
public int Parts { get; set; }
[JsonProperty("includeUploadUrl")]
public bool IncludeUploadUrl { get; set; }
[JsonProperty("type")]
public string Type { get; set; }
}

class BinaryAssets
{
[JsonProperty("binaries")]
public List<BaseBinaryAsset> Binaries { get; set; }

public BinaryAssets()
{
Binaries = new List<BaseBinaryAsset>();
}

public void AddBinary(BaseBinaryAsset binary)
{
Binaries.Add(binary);
}
}
}
49 changes: 49 additions & 0 deletions src/DynamoMLDataPipeline/BinaryReferenceComponent.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
using System.Collections.Generic;
using Newtonsoft.Json;

namespace DynamoMLDataPipeline
{
class BinaryReferenceComponent : Dictionary<string, Dictionary<string, IPropertySet>>
{
private string objectId = "autodesk.data:binary.reference.component-1.0.0";
public string ObjectId { get { return objectId; } }
public BinaryReferenceComponent(string binaryId)
{
var propertyDictionary = new Dictionary<string, IPropertySet>();
propertyDictionary.Add("String", new StringPropertySet(binaryId));
propertyDictionary.Add("Uint32", new IntPropertySet());

this.Add("binary_reference", propertyDictionary);
}
}

class StringPropertySet : IPropertySet
{
[JsonProperty("id")]
public string Id { get; set; }
[JsonProperty("revision")]
public string Revision { get; set; }

public StringPropertySet(string binaryId, string revision = "v0")
{
Id = binaryId;
Revision = revision;
}
}

class IntPropertySet : IPropertySet
{
[JsonProperty("end")]
public int End { get; set; }
[JsonProperty("start")]
public int Start { get; set; }

public IntPropertySet(int start = 0, int end = 8710)
{
End = end;
Start = start;
}
}

interface IPropertySet { }
}
Loading

0 comments on commit 79a3c97

Please sign in to comment.