diff --git a/.gitignore b/.gitignore index 47a090737..28a11c6bd 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ _site RUNNING_PID conf/solr/cores/collins/data .DS_Store -Gemfile.lock .cache-main .cache-tests diff --git a/app/collins/models/LshwHelper.scala b/app/collins/models/LshwHelper.scala index 4a6f27bb2..72df84c8c 100644 --- a/app/collins/models/LshwHelper.scala +++ b/app/collins/models/LshwHelper.scala @@ -213,14 +213,14 @@ object LshwHelper extends CommonHelper[LshwRepresentation] { val baseDescription = amfinder(seq, BaseDescription, _.toString, "") val baseProduct = amfinder(seq, BaseProduct, _.toString, "") val baseVendor = amfinder(seq, BaseVendor, _.toString, "") - val baseSerial = amfinder(seq, BaseSerial, _.toString, "") + val baseSerial = amfinder(seq, BaseSerial, x => if (x.isEmpty) { None } else { Some(x) }, None) Seq(ServerBase(baseDescription, baseProduct, baseVendor, baseSerial)) }.getOrElse(Nil) val filteredMeta = meta.map { case(groupId, metaSeq) => val newSeq = filterNot( metaSeq, - Set(BaseDescription.id, BaseProduct.id, BaseVendor.id) + Set(BaseDescription.id, BaseProduct.id, BaseVendor.id, BaseSerial.id) ) groupId -> newSeq } @@ -229,12 +229,15 @@ object LshwHelper extends CommonHelper[LshwRepresentation] { protected def collectBase(asset: Asset, lshw: LshwRepresentation): Seq[AssetMetaValue] = { val base = lshw.base - Seq( + val expectedAttrs = Seq( AssetMetaValue(asset, BaseDescription.id, base.description), AssetMetaValue(asset, BaseProduct.id, base.product), - AssetMetaValue(asset, BaseVendor.id, base.vendor), - AssetMetaValue(asset, BaseSerial.id, base.serial) + AssetMetaValue(asset, BaseVendor.id, base.vendor) ) + base.serial match { + case Some(x) => expectedAttrs ++ Seq(AssetMetaValue(asset, BaseSerial.id, base.serial.get)) + case None => expectedAttrs + } } } diff --git a/app/collins/models/lshw/ServerBase.scala b/app/collins/models/lshw/ServerBase.scala index eee1378cf..0400ed353 100644 --- a/app/collins/models/lshw/ServerBase.scala +++ b/app/collins/models/lshw/ServerBase.scala @@ -12,7 +12,7 @@ object ServerBase { (json \ "DESCRIPTION").as[String], (json \ "PRODUCT").as[String], (json \ "VENDOR").as[String], - (json \ "SERIAL").as[String])) + (json \ "SERIAL").asOpt[String])) override def writes(serverbase: ServerBase) = JsObject(Seq( "DESCRIPTION" -> Json.toJson(serverbase.description), "PRODUCT" -> Json.toJson(serverbase.product), @@ -22,7 +22,7 @@ object ServerBase { } case class ServerBase( - description: String = "", product: String = "", vendor: String = "", serial: String = "") extends LshwAsset { + description: String = "", product: String = "", vendor: String = "", serial: Option[String] = None) extends LshwAsset { import ServerBase._ override def toJsValue() = Json.toJson(this) diff --git a/app/collins/util/config/LldpConfig.scala b/app/collins/util/config/LldpConfig.scala index 1f1feb4c3..201242434 100644 --- a/app/collins/util/config/LldpConfig.scala +++ b/app/collins/util/config/LldpConfig.scala @@ -5,8 +5,10 @@ object LldpConfig extends Configurable { override val referenceConfigFilename = "lldp_reference.conf" def requireVlanName = getBoolean("requireVlanName", true) + def requireVlanId = getBoolean("requireVlanId", false) override protected def validateConfig() { requireVlanName + requireVlanId } } diff --git a/app/collins/util/parsers/LldpParser.scala b/app/collins/util/parsers/LldpParser.scala index 433bea3dd..cfb9c6ab1 100644 --- a/app/collins/util/parsers/LldpParser.scala +++ b/app/collins/util/parsers/LldpParser.scala @@ -67,16 +67,21 @@ class LldpParser(txt: String) extends CommonParser[LldpRepresentation](txt) { } protected def findVlans(seq: NodeSeq): Seq[Vlan] = { + // TODO(gabe): make this less brittle and handle missing vlan-id (seq \\ "vlan").foldLeft(Seq[Vlan]()) { + // some switches don't report a vlan-id, despite a VLAN being configured + // on an interface. Lets be flexible here and allow it to be empty. case (vseq, vlan) => - val id = Option(vlan \ "@vlan-id" text).filter(_.nonEmpty).getOrElse("") + val idOpt = Option(vlan \ "@vlan-id" text).filter(_.nonEmpty) val name = vlan.text if (LldpConfig.requireVlanName) { - requireNonEmpty((id -> "vlan-id"), (name -> "vlan name")) - } else { - requireNonEmpty((id -> "vlan-id")) + requireNonEmpty((name -> "vlan name")) } - Vlan(id.toInt, name) +: vseq + if (LldpConfig.requireVlanId) { + requireNonEmpty((idOpt.getOrElse("") -> "vlan id")) + } + val id = idOpt.map(_.toInt).getOrElse(0) + Vlan(id, name) +: vseq } } diff --git a/app/collins/util/parsers/LshwParser.scala b/app/collins/util/parsers/LshwParser.scala index d1a12a778..97dbfa0d0 100644 --- a/app/collins/util/parsers/LshwParser.scala +++ b/app/collins/util/parsers/LshwParser.scala @@ -184,13 +184,14 @@ class LshwParser(txt: String) extends CommonParser[LshwRepresentation](txt) { // the correct element if ((elem \ "@class" text).toString == "system") { val asset = getAsset(elem) - val serial = (elem \ "serial" text) + // serial may be missing, so be flexible here and allow it to be absent + val serial = (elem \ "serial" headOption).map(_.text) ServerBase(asset.description, asset.product, asset.vendor, serial) } // To spice things up, sometimes we get $everything // instead of just $everything else if (((elem \ "node") \ "@class" text) == "system") { val asset = getAsset(elem \ "node") - val serial = (elem \ "serial" text) + val serial = (elem \ "serial" headOption).map(_.text) ServerBase(asset.description, asset.product, asset.vendor, serial) } else { throw MalformedAttributeException("Expected root class=system node attribute") diff --git a/app/views/asset/show_overview.scala.html b/app/views/asset/show_overview.scala.html index 03a9e50a5..42d611963 100644 --- a/app/views/asset/show_overview.scala.html +++ b/app/views/asset/show_overview.scala.html @@ -24,13 +24,15 @@

Asset Overview System and user attributes

} @if(aa.addresses.size > 0) { - Ip Addresses + IP Addresses @TagDecorator.decorate("IP_ADDRESS", aa.addresses.map(_.dottedAddress).toList, ", ") Primary IP Addresses } - Asset Tag + + Asset Tag + @if(SoftLayerConfig.enabled && SoftLayer.isSoftLayerAsset(aa.asset)) { @slLink(aa.asset, aa.asset.tag) } else { @@ -40,7 +42,7 @@

Asset Overview System and user attributes

@defining(aa.asset.nodeClass){ nodeclass => - Classification + Classification @nodeclass.map { nc => @@ -60,12 +62,12 @@

Asset Overview System and user attributes

} - Asset Type + Asset Type @aa.asset.assetType.label - Asset Status + Asset Status @aa.asset.getStatusName @if(aa.asset.isMaintenance) { See Notes @aa.asset.status.description @@ -76,7 +78,7 @@

Asset Overview System and user attributes

@if(aa.asset.stateId != 0) { @State.findById(aa.asset.stateId).map { state => - Asset State + Asset State @state.label @state.description @@ -94,7 +96,12 @@

Asset Overview System and user attributes

@MetaValueOrderer.order(aa.mvs.filter(_.getName() != "HOSTNAME")).map { case(size, mv) => - @mv.getLabel() @if(size > 1 || mv.getGroupId() != 0){(@mv.getGroupId())} + + @mv.getLabel() @if(size > 1 || mv.getGroupId() != 0){(@mv.getGroupId())} + + @{ mv.getName match { diff --git a/conf/docker/permissions.yaml b/conf/docker/permissions.yaml index a62b2ec08..d1218bd82 100644 --- a/conf/docker/permissions.yaml +++ b/conf/docker/permissions.yaml @@ -16,6 +16,8 @@ permissions: - "g=infra" - "g=ops" - "g=sre" + feature.canWriteEncryptedTags: + - "u=admins" controllers.Admin: - "g=infra" - "g=ops" @@ -43,6 +45,12 @@ permissions: - "g=sre" - "u=admins" - "u=tools" + controllers.AssetApi.updateAssetStatus: + - "g=infra" + - "g=ops" + - "g=sre" + - "u=admins" + - "u=tools" controllers.AssetApi.updateAssetForMaintenance: - "g=infra" - "g=ops" @@ -105,3 +113,15 @@ permissions: - "g=platform" - "u=admins" - "u=tools" + controllers.IpmiApi.updateIpmi: + - "g=infra" + - "g=ops" + - "g=sre" + - "u=admins" + - "u=tools" + controllers.Firehose.stream: + - "g=infra" + - "g=ops" + - "g=sre" + - "u=admins" + - "u=tools" diff --git a/conf/reference/lldp_reference.conf b/conf/reference/lldp_reference.conf index b9628a066..d92ea7b4f 100644 --- a/conf/reference/lldp_reference.conf +++ b/conf/reference/lldp_reference.conf @@ -2,4 +2,7 @@ lldp { # Refuse to accept lldp information with no name (aka # description) requireVlanName = true + # allow VLANs to omit vlan-id, to support odd devices and layer 2 + # deployments + requireVlanId = false } diff --git a/support/ruby/collins-client/Gemfile.lock b/support/ruby/collins-client/Gemfile.lock new file mode 100644 index 000000000..b6d4cbab6 --- /dev/null +++ b/support/ruby/collins-client/Gemfile.lock @@ -0,0 +1,57 @@ +PATH + remote: . + specs: + collins_client (0.2.19) + httparty (~> 0.11.0) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.0) + public_suffix (~> 2.0, >= 2.0.2) + crack (0.4.3) + safe_yaml (~> 1.0.0) + diff-lcs (1.3) + docile (1.1.5) + hashdiff (0.3.2) + httparty (0.11.0) + multi_json (~> 1.0) + multi_xml (>= 0.5.2) + json (2.0.3) + multi_json (1.12.1) + multi_xml (0.6.0) + public_suffix (2.0.5) + rake (10.5.0) + rspec (2.99.0) + rspec-core (~> 2.99.0) + rspec-expectations (~> 2.99.0) + rspec-mocks (~> 2.99.0) + rspec-core (2.99.2) + rspec-expectations (2.99.2) + diff-lcs (>= 1.1.3, < 2.0) + rspec-mocks (2.99.4) + safe_yaml (1.0.4) + simplecov (0.13.0) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.0) + webmock (1.24.6) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff + yard (0.9.8) + +PLATFORMS + ruby + +DEPENDENCIES + collins_client! + rake (~> 10.4) + rspec (~> 2.99) + simplecov (~> 0.10) + webmock (~> 1.21) + yard (~> 0.8) + +BUNDLED WITH + 1.13.6 diff --git a/support/ruby/collins-notify/collins_notify.gemspec b/support/ruby/collins-notify/collins_notify.gemspec index 4f4b1c889..43473fe39 100644 --- a/support/ruby/collins-notify/collins_notify.gemspec +++ b/support/ruby/collins-notify/collins_notify.gemspec @@ -56,7 +56,7 @@ Gem::Specification.new do |s| s.add_development_dependency('rdoc','~> 3.12') s.add_development_dependency('bundler','>= 1.2.0') s.add_development_dependency('simplecov','~> 0.9.1') - s.add_development_dependency('rake') + s.add_development_dependency('rake', '~> 10.5') end diff --git a/support/ruby/collins-state/Gemfile.lock b/support/ruby/collins-state/Gemfile.lock new file mode 100644 index 000000000..809f371d0 --- /dev/null +++ b/support/ruby/collins-state/Gemfile.lock @@ -0,0 +1,64 @@ +PATH + remote: . + specs: + collins_state (0.2.13) + collins_client (~> 0.2.7) + escape (~> 0.0.4) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.0) + public_suffix (~> 2.0, >= 2.0.2) + collins_client (0.2.19) + httparty (~> 0.11.0) + crack (0.4.3) + safe_yaml (~> 1.0.0) + diff-lcs (1.3) + docile (1.1.5) + escape (0.0.4) + hashdiff (0.3.2) + httparty (0.11.0) + multi_json (~> 1.0) + multi_xml (>= 0.5.2) + json (2.0.3) + multi_json (1.12.1) + multi_xml (0.6.0) + public_suffix (2.0.5) + rake (10.5.0) + redcarpet (3.4.0) + rspec (2.99.0) + rspec-core (~> 2.99.0) + rspec-expectations (~> 2.99.0) + rspec-mocks (~> 2.99.0) + rspec-core (2.99.2) + rspec-expectations (2.99.2) + diff-lcs (>= 1.1.3, < 2.0) + rspec-mocks (2.99.4) + safe_yaml (1.0.4) + simplecov (0.13.0) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.0) + webmock (1.24.6) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff + yard (0.8.7.6) + +PLATFORMS + ruby + +DEPENDENCIES + bundler + collins_state! + rake (~> 10.5) + redcarpet (~> 3.2) + rspec (~> 2.99) + simplecov + webmock (~> 1.21) + yard (~> 0.8.7) + +BUNDLED WITH + 1.13.6 diff --git a/support/ruby/collins-state/collins_state.gemspec b/support/ruby/collins-state/collins_state.gemspec index 5229619fe..711a764d2 100644 --- a/support/ruby/collins-state/collins_state.gemspec +++ b/support/ruby/collins-state/collins_state.gemspec @@ -34,9 +34,9 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'escape', '~> 0.0.4' s.add_development_dependency 'rspec', '~> 2.99' - s.add_development_dependency 'yard' - s.add_development_dependency 'redcarpet' - s.add_development_dependency 'webmock' + s.add_development_dependency 'yard', '~> 0.8.7' + s.add_development_dependency 'redcarpet', '~> 3.2' + s.add_development_dependency 'webmock', '~> 1.21' s.add_development_dependency 'bundler' s.add_development_dependency 'simplecov' s.add_development_dependency 'rake', '~> 10.5' diff --git a/test/collins/util/parsers/LldpParserSpec.scala b/test/collins/util/parsers/LldpParserSpec.scala index b5e9b9f26..8a26b9916 100644 --- a/test/collins/util/parsers/LldpParserSpec.scala +++ b/test/collins/util/parsers/LldpParserSpec.scala @@ -58,6 +58,21 @@ class LldpParserSpec extends mutable.Specification { } } + "missing vlan-id ok when !lldp.requireVlanId" in new LldpParserHelper("lldpctl-no-vlan-id.xml", Map("lldp.requireVlanId" -> "false")){ + val parseResult = parsed() + parseResult must beRight + parseResult.right.toOption must beSome.which { rep => + rep.interfaceCount mustEqual (1) + rep.vlanNames.toSet mustEqual (Set("fake")) + rep.vlanIds.toSet mustEqual (Set(0)) + } + } + "missing vlan-id not ok when lldp.requireVlanId" in new LldpParserHelper("lldpctl-no-vlan-id.xml", Map("lldp.requireVlanId" -> "true")){ + val parseResult = parsed() + parseResult must beLeft + } + + "Parse XML with four network interfaces" in new LldpParserHelper("lldpctl-four-nic.xml") { val parseResult = parsed() parseResult must beRight @@ -74,6 +89,20 @@ class LldpParserSpec extends mutable.Specification { } } + "Parse XML with optional fields" in { + "Missing vlan-id" in new LldpParserHelper("lldpctl-bad.xml", Map("lldp.requireVlanId" -> "false")) { + // missing vlan-id is acceptable, for compatibility with odd switches that + // do not report a vlan-id despite being configured + val invalidXml = getResource(filename) + override def getParseResults(data: String): Either[Throwable, LldpRepresentation] = { + getParser(data).parse() + } + val s = """DFW-LOGGING""" + val r = """DFW-LOGGING""" + getParseResults(invalidXml.replace(s, r)) must beRight + } + } + "Parse a generated XML file" in new LldpParserHelper("lldpctl-empty.xml") { parsed() must beRight } @@ -163,15 +192,7 @@ class LldpParserSpec extends mutable.Specification { val r = """""" getParseResults(invalidXml.replace(s, r)) must beLeft } - "Missing vlan id" in new LldpParserHelper("lldpctl-bad.xml") { - val invalidXml = getResource(filename) - override def getParseResults(data: String): Either[Throwable, LldpRepresentation] = { - getParser(data).parse() - } - val s = """DFW-LOGGING""" - val r = """DFW-LOGGING""" - getParseResults(invalidXml.replace(s, r)) must beLeft - } + } } diff --git a/test/collins/util/parsers/LshwParserSpec.scala b/test/collins/util/parsers/LshwParserSpec.scala index 4c770cffd..71d59d571 100644 --- a/test/collins/util/parsers/LshwParserSpec.scala +++ b/test/collins/util/parsers/LshwParserSpec.scala @@ -50,7 +50,7 @@ class LshwParserSpec extends mutable.Specification { rep.base.product mustEqual "PowerEdge C6105 (N/A)" rep.base.vendor mustEqual "Winbond Electronics" - rep.base.serial mustEqual "FZ1NXQ1" + rep.base.serial.get mustEqual "FZ1NXQ1" } } // with a 10-gig card @@ -386,6 +386,19 @@ class LshwParserSpec extends mutable.Specification { } // Parse softlayer supermicro (Intel) lshw output" + "Handle missing fields in LSHW" in { + "No base serial" in new LshwParserHelper("missing-base-serial.xml") { + val parseResults = parsed() + parseResults must beRight + parseResults.right.toOption must beSome.which { rep => + rep.base.product mustEqual "Virtual Machine (None)" + rep.base.vendor mustEqual "Microsoft Corporation" + rep.base.serial mustEqual None + rep.base.description mustEqual "Desktop Computer" + } + } + } + "Parse Dell LSHW Output" in { "R620 LSHW Output" in new LshwParserHelper("lshw-dell-r620-single-cpu.xml") { val parseResults = parsed() diff --git a/test/resources/lldpctl-no-vlan-id.xml b/test/resources/lldpctl-no-vlan-id.xml new file mode 100644 index 000000000..768263495 --- /dev/null +++ b/test/resources/lldpctl-no-vlan-id.xml @@ -0,0 +1,29 @@ + + + + + ??? + system name + system desc + + + + + Port ID here + Port Description Here + 1514 + + + unknown + + + fake + + Network Connectivity Device + + + + + + + diff --git a/test/resources/missing-base-serial.xml b/test/resources/missing-base-serial.xml new file mode 100644 index 000000000..18eb4e6ca --- /dev/null +++ b/test/resources/missing-base-serial.xml @@ -0,0 +1,259 @@ + + + + + + + + Desktop Computer + Virtual Machine (None) + Microsoft Corporation + Hyper-V UEFI Release v1.0 + 0574-2207-1613-4261-5019-1454-34 + 64 + + + + + + + + + SMBIOS version 2.4 + DMI version 2.4 + 32-bit processes + + + Motherboard + Virtual Machine + Microsoft Corporation + 0 + Hyper-V UEFI Release v1.0 + 0574-2207-1613-4261-5019-1454-34 + Virtual Machine + + BIOS + Microsoft Corporation + 0 + Hyper-V UEFI Release v1.0 + 11/26/2012 + 1048576 + + ACPI + + + + CPU + Xeon (None) + Intel Corp. + 4 + cpu@0 + Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz + None + None + 2400000000 + 4000000000 + 64 + 3705032704 + + mathematical co-processor + FPU exceptions reporting + + virtual mode extensions + debugging extensions + page size extensions + time stamp counter + model-specific registers + 4GB+ memory addressing (Physical Address Extension) + machine check exceptions + compare and exchange 8-byte + on-chip advanced programmable interrupt controller (APIC) + fast system calls + memory type range registers + page global enable + machine check architecture + conditional move instruction + page attribute table + 36-bit page size extensions + + multimedia extensions (MMX) + fast floating point save/restore + streaming SIMD extensions (SSE) + streaming SIMD extensions (SSE2) + self-snoop + fast system calls + no-execute bit (NX) + 64bits extensions (x86-64) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + System Memory + 6 + System board or motherboard + 4294967296 + + None + Microsoft Corporation + 0 + None + M00 + 4160749568 + + + None + Microsoft Corporation + 1 + None + M01 + 134217728 + + + + 1 + scsi0 + + SCSI Disk + 0.0.0 + scsi@0:0.0.0 + /dev/sda + 8:0 + 53687091200 + + + + + + + GUID Partition Table version 1.00 + Partitioned disk + GUID partition table + + + Windows FAT volume + mkfs.fat + 1 + scsi@0:0.0.0,1 + FAT16 + 32c4-96a4 + 209714688 + 209714688 + + + + + + + Contains boot code + Windows FAT + initialized volume + + + + data partition + Windows + 2 + scsi@0:0.0.0,2 + c1747320-6774-47dc-8e61-104051115270 + 524287488 + + + LVM Physical Volume + Linux + 3 + scsi@0:0.0.0,3 + /dev/sda3 + 8:3 + tpeGWj-gIj7-Xpa7-22j6-LqZf-qP6k-ayZLO1 + 52950990848 + + Multi-volumes + + + + + + SCSI CD-ROM + 0.0.1 + scsi@0:0.0.1 + /dev/cdrom + /dev/sr0 + 11:0 + + + + + Audio CD playback + + + + + + Ethernet interface + 1 + veth7720825 + e2:89:fc:fa:6c:2c + 10000000000 + + + + + + + + + + + + + + Physical interface + + + + Ethernet interface + 2 + eth0 + 00:1d:d8:b7:1e:e7 + + + + + + + + + + + Physical interface + + + +