lsf-full.txt

[cluster lsf]
    FormLayout = selectionpanel
    Category = Schedulers

    [[node defaults]]
        UsePublicNetwork = $UsePublicNetwork
        Credentials = $Credentials
        Region = $Region
        KeyPairLocation = ~/.ssh/cyclecloud.pem
        ImageName = cycle.image.centos7
        SubnetId = $ComputeSubnet


        CloudInit = '''#cloud-config
packages:
- libnsl
        '''

        [[[configuration]]]

        cuser.base_home_dir = /shared/home

        lsf.lsf_top = $LSF_TOP  # /grid/lsf
        lsf.lsf_logdir = ${LSF_TOP}/log
        lsf.admin.home = /shared/home/lsfadmin
        lsf.entitled_install = true 
        lsf.shared_install = true
        lsf.cyclecloud_cluster_name = $WorkerCluster
        lsf.custom_script_uri = $customScriptUri
        lsf.required_patch_version = 600488 

        cyclecloud.selinux.policy = permissive

        # Disable normal NFS exports and mounts
        cyclecloud.mounts.sched.disabled = true
        cyclecloud.mounts.shared.disabled = true
        cyclecloud.exports.sched.disabled = true
        cyclecloud.exports.shared.disabled = true
        cyclecloud.exports.sched.samba.enabled = false
        cyclecloud.exports.shared.samba.enabled = false
        cyclecloud.exports.defaults.samba.enabled = false      
        cshared.server.legacy_links_disabled = true

        [[[cluster-init cyclecloud/lsf:default]]]

        [[[configuration cyclecloud.mounts.nfs_shared]]]
        type = nfs
        mountpoint = /shared
        export_path = $NFSSharedExportPath
        address = $NFSAddress
        options = $NFSSharedMountOptions

        [[[configuration cyclecloud.mounts.nfs_sched]]]
        type = nfs
        mountpoint = /sched
        disabled = $NFSSchedDisable

        [[[configuration cyclecloud.mounts.additional_nfs]]]
        disabled = ${AdditionalNAS isnt true}
        type = nfs
        address = ${ifThenElse(AdditionalNAS, AdditonalNFSAddress, undefined)}
        mountpoint = ${ifThenElse(AdditionalNAS, AdditionalNFSMountPoint, undefined)}
        export_path = ${ifThenElse(AdditionalNAS, AdditionalNFSExportPath, undefined)}
        options = ${ifThenElse(AdditionalNAS, AdditionalNFSMountOptions, undefined)}


    [[node scheduler]]
        MachineType = $MasterMachineType
        ImageName = $SchedulerImageName
        InitialCount = $MasterNodeCountHA
        AdditionalClusterInitSpecs = $MasterClusterInitSpecs

        [[[configuration]]]
        cyclecloud.mounts.nfs_sched.disabled = true
        cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"}
        run_list = role[scheduler],recipe[cyclecloud],recipe[cshared::directories],recipe[cuser],recipe[cshared::server],recipe[lsf::install],recipe[lsf::master]
        lsf.num_placement_groups = $MaxNumScalesets

        [[[network-interface eth0]]]
        AssociatePublicIpAddress = $UsePublicNetwork

        [[[volume sched]]]
        Size = 30
        SSD = True
        Mount = builtinsched
        Persistent = False

        [[[volume shared]]]
        Size = ${ifThenElse(NFSType == "Builtin", FilesystemSize, 2)}
        SSD = True
        Mount = builtinshared
        Persistent = ${NFSType == "Builtin"}

        [[[configuration cyclecloud.mounts.builtinsched]]]
        disabled = ${NFSType != "Builtin"}
        mountpoint = /sched
        fs_type = xfs

        [[[configuration cyclecloud.mounts.builtinshared]]]
        disabled = ${NFSType != "Builtin"}
        mountpoint = /shared
        fs_type = xfs

        [[[configuration cyclecloud.exports.builtinsched]]]
        disabled = ${NFSSchedDisable}
        export_path = /sched
        options = no_root_squash
        samba.enabled = false
        type = nfs

        [[[configuration cyclecloud.exports.builtinshared]]]
        disabled = ${NFSType != "Builtin"}
        export_path = /shared
        samba.enabled = false
        type = nfs

        [[[cluster-init cyclecloud/lsf:master]]]


    [[node proxy]]
        IsReturnProxy = $ReturnProxy
        MachineType = $ProxyMachineType
        SubnetId = ${ifThenElse(AccessSubnet !== undefined, AccessSubnet, ComputeSubnet)}
        CloudInit = ''
        [[[configuration]]]
        run_list =  recipe[cganglia::server]
        cyclecloud.discoverable = true
        cuser.base_home_dir = /home

        [[[network-interface eth0]]]
        AssociatePublicIpAddress = $ReturnProxy

        [[[input-endpoint ganglia]]]
        PrivatePort = 8652
        PublicPort = 8652

        [[[input-endpoint SSH]]]
        PrivatePort = 22
        PublicPort = 22

    [[nodearray ondemand]]
        Azure.MaxScalesetSize = $MaxScalesetSize
        MachineType = $ExecuteMachineType
        AdditionalClusterInitSpecs = $ExecuteClusterInitSpecs
        Interruptible = False
        ImageName = $ImageName

        MaxCoreCount = $MaxExecuteCoreCount

        [[[configuration]]]
        run_list = recipe[cyclecloud],recipe[cshared::client],recipe[cuser],recipe[cganglia::client],recipe[lsf::worker]
        cyclecloud.maintenance_converge.enabled = false
        lsf.attribute_names = nodearray
        lsf.attributes.nodearray = ondemand

        [[[cluster-init cyclecloud/lsf:execute]]]
        
    [[nodearray ondemandmpi]]
    	Extends = ondemand
        PlacementGroupId = ondemandmpi-manual 

        [[[configuration]]]
        lsf.attribute_names = nodearray cyclecloudmpi placementgroup
        lsf.attributes.nodearray = ondemandmpi
        lsf.attributes.cyclecloudmpi = true
        lsf.attributes.placementgroup = ondemandmpi-manual

    [[nodearray lowprio]]
        Extends = ondemand
        Interruptible = true
        MaxPrice = -1


        [[[configuration]]]
        lsf.attribute_names = nodearray cyclecloudlowprio
        lsf.attributes.nodearray = lowprio
        lsf.attributes.cyclecloudlowprio = true

    [[nodearray gpu]]
        Extends = ondemand
        ImageName = $GPUImageName
        MachineType = $GPUMachineType
        AdditionalClusterInitSpecs = $GPUClusterInitSpecs
        [[[configuration]]]
        lsf.attribute_names = nodearray
        lsf.attributes.nodearray = gpu

    [[nodearray gpumpi]]
    	Extends = gpu
        PlacementGroupId = gpumpi-manual 

        [[[configuration]]]
        lsf.attribute_names = nodearray cyclecloudmpi placementgroup
        lsf.attributes.nodearray = gpu
        lsf.attributes.cyclecloudmpi = true
        lsf.attributes.placementgroup = gpumpi-manual

    # nodes will start in closed_RC state 
    [[nodearray submit]] 
        Priority = 600 
        Extends = ondemand 
        MachineType = $SubmitMachineType 

        [[[configuration]]] 
        lsf.custom_script_uri = "file:///mnt/cluster-init/lsf/execute/files/user_data-full.sh" 

        
[parameters About]
Order = 1

    [[parameters About LSF]]

        [[[parameter LSF]]]
        HideLabel = true
        Config.Plugin = pico.widget.HtmlTemplateWidget
        Config.Template := "IBM Spectrum LSF is a complete workload management solution for demanding HPC environments"

        [[[parameter Readme]]]
        HideLabel = true
        Config.Plugin = pico.widget.HtmlTemplateWidget
        Config.Template = "Follow the instructions in the <a href="https://github.com/Azure/cyclecloud-lsf" target=\"_blank\">README</a> for details on instructions on extending and configuring the Project for your environment."


[parameters Required Settings]
Order = 10

    [[parameters Cloud Service Provider Configuration]]
    Description = Configure the Cloud Provider account options.
    Order = 10

        [[[parameter Region]]]
        Label = Region
        Description = Deployment Location
        ParameterType = Cloud.Region

    [[parameters Compute Configurations]]
    Description = "Configure the execute array for the cluster.  VM sizes and autoscaling limits."
    Order = 20

        [[[parameter ExecuteMachineType]]]
        Label = Compute Type
        Description = The machine type for execute nodes
        ParameterType = Cloud.MachineType
        DefaultValue = Standard_D4_v5 
        Config.Multiselect = true

        [[[parameter GPUMachineType]]]
        Label = GPU Type
        Description = The machine type for GPU nodes
        ParameterType = Cloud.MachineType
        DefaultValue = Standard_NC6s_v3
        Config.Multiselect = false

        [[[parameter SubmitMachineType]]]
        Label = Compute Type
        Description = The machine type for execute nodes
        ParameterType = Cloud.MachineType
        DefaultValue = Standard_D4_v5 
        Config.Multiselect = true

        [[[parameter MaxExecuteCoreCount]]]
        Label = Max Cores
        Description = The total number of execute cores to start
        DefaultValue = 500
        Config.Plugin = pico.form.NumberTextBox
        Config.MinValue = 1
        Config.MaxValue = 5000
        Config.IntegerOnly = true


    [[parameters Networking]]
    Description = Networking settings
    Order = 40

        [[[parameter ComputeSubnet]]]
        Label = Compute Subnet
        Description = Subnet Resource Path (ResourceGroup/VirtualNetwork/Subnet)
        Required = true
        ParameterType = Azure.Subnet

    [[parameters Advanced Networking]]
    Description = Advanced networking settings

        [[[parameter ReturnProxy]]]
        Label = Return Proxy
        DefaultValue = true
        ParameterType = Boolean
        Config.Label = Use SSH tunnel to connect to CycleCloud (required if direct access is blocked)

        [[[parameter UsePublicNetwork]]]
        Label = Public Head Node
        DefaultValue = true
        ParameterType = Boolean
        Config.Label = Access scheduler node from the Internet

        [[[parameter ExecuteNodesPublic]]]
        Label = Public Execute
        DefaultValue = false
        ParameterType = Boolean
        Config.Label = Access execute nodes from the Internet
        Conditions.Excluded := UsePublicNetwork isnt true

[parameters Advanced Settings]
Order = 20

    [[parameters Azure Settings]]
    Description = Provider Account Name
    Order = 10 

        [[[parameter Credentials]]]
        Description = The credentials for the cloud provider
        ParameterType = Cloud.Credentials

    [[ parameters Lsf Configuration ]]
    Description = "The compute nodes should have LSF installed, set the location of the installation."
    Order = 15

        [[[parameter lsf_top]]]
        Label = lsf_top
        DefaultValue = "/sched/lsf"
        Description = Location of LSF install on base image.

        [[[parameter customScriptUri]]]
        Label = Custom Script URI
        DefaultValue = "file:///mnt/cluster-init/lsf/execute/files/user_data-full.sh"
        Description = Script run at node boot up with LSF context.

        [[[parameter WorkerCluster]]]
        Label = Worker Cluster Name
        DefaultValue := undefined
        Description = Set if starting nodes in a different cluster

    [[parameters Software]]
    Description = "Specify the scheduling software, and base OS installed on all nodes, and optionally the cluster-init and chef versions from your Locker."
    Order = 20

        [[[parameter SchedulerImageName]]]
        Label = Scheduler OS
        ParameterType = Cloud.Image
        Config.OS = linux
        DefaultValue = almalinux:almalinux-hpc:8-hpc-gen2:latest
        #Config.Filter := Package in {"cycle.image.centos7"}

        [[[parameter ImageName]]]
        Label = Execute OS
        ParameterType = Cloud.Image
        Config.OS = linux
        DefaultValue = almalinux:almalinux-hpc:8-hpc-gen2:latest
        #Config.Filter := Package in {"cycle.image.centos7"}

        [[[parameter GPUImageName]]]
        Label = GPU Base OS
        ParameterType = Cloud.Image
        Config.OS = linux
        DefaultValue = almalinux:almalinux-hpc:8-hpc-gen2:latest
        #Config.Filter := Package in {"cycle.image.centos7"}

        [[[parameter MasterClusterInitSpecs]]]
        Label = Master Cluster-Init
        DefaultValue = =undefined
        Description = Cluster init specs to apply to the master node
        ParameterType = Cloud.ClusterInitSpecs
    
        [[[parameter ExecuteClusterInitSpecs]]]
        Label = Execute Cluster-Init
        DefaultValue = =undefined
        Description = Cluster init specs to apply to non-GPU execute nodes
        ParameterType = Cloud.ClusterInitSpecs

        [[[parameter GPUClusterInitSpecs]]]
        Label = GPU Cluster-Init
        DefaultValue = =undefined
        Description = Cluster init specs to apply to GPU nodes only
        ParameterType = Cloud.ClusterInitSpecs

    [[parameters Master Nodes]]
    Description = "Optional master Fail-over configuration"
    Order = 30

        [[[parameter MasterMachineType]]]
        Label = Master Type
        Description = The machine type for HA master array.
        ParameterType = Cloud.MachineType
        DefaultValue = Standard_D2_v5

        [[[parameter MasterNodeCountHA]]]
        Label = Master Count
        Description = Node count for HighAvailability Master
        DefaultValue = 2
        Config.Plugin = pico.form.NumberTextBox
        Config.MinValue = 0
        Config.MaxValue = 2
        Config.IntegerOnly = true

    [[parameters Advanced Networking]]
    Description = Advanced networking settings
    Order = 35


        [[[parameter AccessSubnet]]]
        Label = Access Subnet
        Required = false
        ParameterType = Azure.Subnet
        Description = Select access subnet or use default.

        [[[parameter ReturnProxy]]]
        Label = Return Proxy
        DefaultValue = true
        ParameterType = Boolean
        Config.Label = Use SSH tunnel to connect to CycleCloud (required if direct access is blocked)

    [[parameters Advanced Machines]]
    Description = Additional machine configurations
    Order = 40

        [[[parameter FSMachineType]]]
        Label = Fileserver Type
        Description = The machine type for shared filer.
        ParameterType = Cloud.MachineType
        DefaultValue = Standard_D4s_v5

        [[[parameter ProjVolSize]]]
        Label = Size (GB)
        Description = FileServer Constituent RAID Volume Size (Count: 5)
        DefaultValue = 200
        Config.Plugin = pico.form.NumberTextBox
        Config.MinValue = 20
        Config.MaxValue = 4095
        Config.IntegerOnly = true

        [[[parameter ProxyMachineType]]]
        Label = Proxy Type
        Description = The machine type for proxy.
        ParameterType = Cloud.MachineType
        DefaultValue = Standard_D2_v5

        [[[parameter MaxScalesetSize]]]
        Label = Max VMs in VMSS
        Description = Max number of VMs in a VMSS
        DefaultValue = 40
        Config.Plugin = pico.form.NumberTextBox
        Config.MinValue = 1
        Config.MaxValue = 1000
        Config.IntegerOnly = true

        [[[parameter MaxNumScalesets]]]
        Label = Max VMSS count
        Description = Max number of VMSS that the RC can allocate.
        DefaultValue = 1
        Config.Plugin = pico.form.NumberTextBox
        Config.MinValue = 0
        Config.MaxValue = 500
        Config.IntegerOnly = true


[parameters Network Attached Storage]
Order = 15


    [[parameters Scheduler Mount]]
    Order = 5
        [[[parameter About sched]]]
        HideLabel = true
        Config.Plugin = pico.widget.HtmlTemplateWidget
        Config.Template = ''' <p>The directory <code>/sched</code> is a network attached mount and exists in all nodes of the cluster. 
            Slurm's configuration is linked in from this directory. It's managed by the scheduler node. 
            To disable the mount of the /sched directory, and to supply your own for a <strong>hybrid scenario</strong>, select the checkbox below '''
        Order = 6

        [[[parameter NFSSchedDisable]]]
        HideLabel = true
        DefaultValue = false
        Widget.Plugin = pico.form.BooleanCheckBox
        Widget.Label = External Scheduler

    [[parameters Default NFS Share]]
    Order = 10
        [[[parameter About shared]]]
        HideLabel = true
        Config.Plugin = pico.widget.HtmlTemplateWidget
        Config.Template := "<p>The directory <code>/shared</code> is a network attached mount and exists in all nodes of the cluster. Users' home directories reside within this mountpoint with the base homedir <code>/shared/home</code>.<br><br>There are two options for providing this mount:<br> <strong>[Builtin]</strong>: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster.<br> <strong>[External NFS]</strong>: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server, provides the mountpoint.</p>"
        Order = 20

        [[[parameter NFSType]]]
        Label = NFS Type
        ParameterType = StringList
        Config.Label = Type of NFS to use for this cluster
        Config.Plugin = pico.form.Dropdown
        Config.Entries := {[Label="External NFS"; Value="External"], [Label="Builtin"; Value="Builtin"]}
        DefaultValue = Builtin

        [[[parameter NFSAddress]]]
        Label = NFS IP Address
        Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache.
        Config.ParameterType = String
        Conditions.Hidden := NFSType != "External"

        [[[parameter NFSSharedExportPath]]]
        Label = Shared Export Path
        Description = The path exported by the file system
        DefaultValue = /shared
        Conditions.Hidden := NFSType != "External"

        [[[parameter NFSSharedMountOptions]]]
        Label = NFS Mount Options
        Description = NFS Client Mount Options
        Conditions.Hidden := NFSType != "External"

        [[[parameter FilesystemSize]]]
        Label = Size (GB)
        Description = The filesystem size
        DefaultValue = 100

        Config.Plugin = pico.form.NumberTextBox
        Config.MinValue = 10
        Config.MaxValue = 10240
        Config.IntegerOnly = true
        Conditions.Excluded := NFSType != "Builtin"

    [[parameters Additional NFS Mount]]
    Order = 20
        [[[parameter Additional NFS Mount Readme]]]
        HideLabel = true
        Config.Plugin = pico.widget.HtmlTemplateWidget
        Config.Template := "<p>Mount another NFS endpoint on the cluster nodes</p>"
        Order = 20

        [[[parameter AdditionalNAS]]]
        HideLabel = true
        DefaultValue = false
        Widget.Plugin = pico.form.BooleanCheckBox
        Widget.Label = Add NFS mount

        [[[parameter AdditonalNFSAddress]]]
        Label = NFS IP Address 
        Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache.
        Config.ParameterType = String
        Conditions.Excluded := AdditionalNAS isnt true

        [[[parameter AdditionalNFSMountPoint]]]
        Label = NFS Mount Point
        Description = The path at which to mount the Filesystem
        DefaultValue = /data
        Conditions.Excluded := AdditionalNAS isnt true

        [[[parameter AdditionalNFSExportPath]]]
        Label = NFS Export Path
        Description = The path exported by the file system
        DefaultValue = /data
        Conditions.Excluded := AdditionalNAS isnt true

        [[[parameter AdditionalNFSMountOptions]]]
        Label = NFS Mount Options
        Description = NFS Client Mount Options
        Conditions.Excluded := AdditionalNAS isnt true