From e612f39403e01e1b43b3acfb95c1501febeaf1c6 Mon Sep 17 00:00:00 2001 From: hsong-rh Date: Fri, 1 Dec 2017 10:18:06 -0500 Subject: [PATCH 1/3] Add valid_encoding checking before insert into xml object. https://bugzilla.redhat.com/show_bug.cgi?id=1518249 --- lib/gems/pending/util/xml/miq_rexml.rb | 4 ++-- lib/gems/pending/util/xml/xml_utils.rb | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/gems/pending/util/xml/miq_rexml.rb b/lib/gems/pending/util/xml/miq_rexml.rb index 45757331e..dff92c54c 100644 --- a/lib/gems/pending/util/xml/miq_rexml.rb +++ b/lib/gems/pending/util/xml/miq_rexml.rb @@ -148,7 +148,7 @@ def initialize(first, second = nil, parent = nil) rescue => err if err.class == ::Encoding::CompatibilityError second_utf8 = second.to_s.force_encoding('UTF-8') - @value = Text.unnormalize(second_utf8, nil) + @value = Text.unnormalize(second_utf8.valid_encoding? ? second_utf8 : "Invalid encoding found", nil) else $log.error "Encoding error: #{second_utf8}" if $log end @@ -223,7 +223,7 @@ def to_xml def add_element(element, attrs = nil) return add_element_orig(element) if element.kind_of?(REXML::Element) attrs.delete_if { |_k, v| v.nil? } unless attrs.nil? - add_element_orig(element.to_s, XmlHelpers.stringify_keys(attrs)) + add_element_orig(element.to_s, XmlHelpers.validate_attrs(attrs)) end alias_method :add_attribute_orig, :add_attribute diff --git a/lib/gems/pending/util/xml/xml_utils.rb b/lib/gems/pending/util/xml/xml_utils.rb index 2968c113a..a9398827a 100644 --- a/lib/gems/pending/util/xml/xml_utils.rb +++ b/lib/gems/pending/util/xml/xml_utils.rb @@ -62,6 +62,12 @@ def self.findElementInt(paths, ele) end class XmlHelpers + # Validate attributes before inserting into xml + def self.validate_attrs(h) + return nil if h.nil? + h.inject({}) { |options, (key, value)| options[key.to_s] = value.to_s.force_encoding('UTF-8').valid_encoding? ? value : "Invalid encoding found"; options } + end + def self.stringify_keys(h) return nil if h.nil? h.inject({}) { |options, (key, value)| options[key.to_s] = value; options } From ed7cb35853c3127de33d55f1d4dc6cfe012ac3a7 Mon Sep 17 00:00:00 2001 From: hsong-rh Date: Thu, 7 Dec 2017 15:17:49 -0500 Subject: [PATCH 2/3] add filter to remove special chars --- lib/gems/pending/util/xml/miq_rexml.rb | 4 ++-- lib/gems/pending/util/xml/xml_utils.rb | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/gems/pending/util/xml/miq_rexml.rb b/lib/gems/pending/util/xml/miq_rexml.rb index dff92c54c..8085123e4 100644 --- a/lib/gems/pending/util/xml/miq_rexml.rb +++ b/lib/gems/pending/util/xml/miq_rexml.rb @@ -147,8 +147,8 @@ def initialize(first, second = nil, parent = nil) @value = Text.unnormalize(second.to_s, nil) rescue => err if err.class == ::Encoding::CompatibilityError - second_utf8 = second.to_s.force_encoding('UTF-8') - @value = Text.unnormalize(second_utf8.valid_encoding? ? second_utf8 : "Invalid encoding found", nil) + second_utf8 = second.to_s.force_encoding('UTF-8').encode('UTF-8', :invalid => :replace, :undef => :replace, :replace => '') + @value = Text.unnormalize(second_utf8) else $log.error "Encoding error: #{second_utf8}" if $log end diff --git a/lib/gems/pending/util/xml/xml_utils.rb b/lib/gems/pending/util/xml/xml_utils.rb index a9398827a..06a38de50 100644 --- a/lib/gems/pending/util/xml/xml_utils.rb +++ b/lib/gems/pending/util/xml/xml_utils.rb @@ -62,10 +62,28 @@ def self.findElementInt(paths, ele) end class XmlHelpers + # reference from REXML::TEXT + VALID_CHAR = [ + 0x9, 0xA, 0xD, + (0x20..0xD7FF), + (0xE000..0xFFFD), + (0x10000..0x10FFFF) + ] + + def self.remove_invalid_chars(str) + str.chars.reject do |c| + case c.ord + when *VALID_CHAR + else + '' + end + end.join + end + # Validate attributes before inserting into xml def self.validate_attrs(h) return nil if h.nil? - h.inject({}) { |options, (key, value)| options[key.to_s] = value.to_s.force_encoding('UTF-8').valid_encoding? ? value : "Invalid encoding found"; options } + h.each_with_object({}) { |(k, v), h| h[k.to_s] = remove_invalid_chars(v.to_s.encode('UTF-8', :undef => :replace, :invalid => :replace, :replace => '')) } end def self.stringify_keys(h) From 2d76f13f90250f25bb9c74d70073f5615dd9cc3d Mon Sep 17 00:00:00 2001 From: hsong-rh Date: Mon, 11 Dec 2017 11:52:26 -0500 Subject: [PATCH 3/3] Add spec test --- lib/gems/pending/util/xml/xml_utils.rb | 11 ++--------- spec/util/xml/miq_rexml_spec.rb | 7 +++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/gems/pending/util/xml/xml_utils.rb b/lib/gems/pending/util/xml/xml_utils.rb index 06a38de50..31ef908b8 100644 --- a/lib/gems/pending/util/xml/xml_utils.rb +++ b/lib/gems/pending/util/xml/xml_utils.rb @@ -62,18 +62,10 @@ def self.findElementInt(paths, ele) end class XmlHelpers - # reference from REXML::TEXT - VALID_CHAR = [ - 0x9, 0xA, 0xD, - (0x20..0xD7FF), - (0xE000..0xFFFD), - (0x10000..0x10FFFF) - ] - def self.remove_invalid_chars(str) str.chars.reject do |c| case c.ord - when *VALID_CHAR + when *REXML::Text::VALID_CHAR else '' end @@ -93,6 +85,7 @@ def self.stringify_keys(h) end class Xml2tags + def self.walk(node, parents = "") tags = [] diff --git a/spec/util/xml/miq_rexml_spec.rb b/spec/util/xml/miq_rexml_spec.rb index 1118e50c1..b19e2ef28 100644 --- a/spec/util/xml/miq_rexml_spec.rb +++ b/spec/util/xml/miq_rexml_spec.rb @@ -27,4 +27,11 @@ xml = MiqXml.load(doc_text) expect(xml.root.elements[1].attributes['attr1']).to eq(attr_string) end + + it "add_element with control characters" do + attr_hash = { "attr1" => "test\u000Fst\u001Fring" } + doc = MiqXml.createDoc(nil) + doc.add_element('element_1', attr_hash) + expect(doc.elements[1].attributes['attr1']).to eq("teststring") + end end