Add YANG lexer

lib/rouge/demos/yang

@@ -0,0 +1,17 @@
+module petstore {
+    namespace "http://gribok.org/ns/yang/example";
+    prefix ex;
+
+    revision 2020-04-01 {
+        description "Example yang";
+    }
+
+    container pets {
+        list dogs {
+            key name;
+            leaf name {
+                type string;
+            }
+        }
+    }
+}

lib/rouge/lexers/yang.rb

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class YANG < RegexLexer
+      title 'YANG'
+      desc "Lexer for the YANG 1.1 modeling language (RFC7950)"
+      tag 'yang'
+      filenames '*.yang'
+      mimetypes 'application/yang'
+
+      id = /[\w_-]+(?=[^\w\-\:])\b/
+
+      #Keywords from RFC7950 ; oriented at BNF style
+      def self.top_stmts_keywords
+        @top_stms_keywords ||= Set.new %w(
+          module submodule
+        )
+      end
+
+      def self.module_header_stmts_keywords
+        @module_header_stmts_keywords ||= Set.new %w(
+          yang-version namespace prefix belongs-to
+        )
+      end
+
+      def self.meta_stmts_keywords
+        @meta_stmts_keywords ||= Set.new %w(
+          organization contact description reference revision
+        )
+      end
+
+      def self.linkage_stmts_keywords
+        @linkage_stmts_keywords ||= Set.new %w(
+          import include revision-date
+        )
+      end
+
+      def self.body_stmts_keywords
+        @body_stms_keywords ||= Set.new %w(
+          extension feature identity typedef grouping augment rpc notification
+          deviation action argument identity if-feature input output
+        )
+      end
+
+      def self.data_def_stmts_keywords
+        @data_def_stms_keywords ||= Set.new %w(
+          container leaf-list leaf list choice anydata anyxml uses case config
+          deviate must when presence refine
+        )
+      end
+
+      def self.type_stmts_keywords
+        @type_stmts_keywords ||= Set.new %w(
+          type units default status bit enum error-app-tag error-message
+          fraction-digits length min-elements max-elements modifier ordered-by
+          path pattern position range require-instance value yin-element base
+        )
+      end
+
+      def self.list_stmts_keywords
+        @list_stmts_keywords ||= Set.new %w(
+          key mandatory unique
+        )
+      end
+
+      #RFC7950 other keywords
+      def self.constants_keywords
+        @constants_keywords ||= Set.new %w(
+          true false current obsolete deprecated add delete replace
+          not-supported invert-match max min unbounded user
+        )
+      end
+
+      #RFC7950 Built-In Types
+      def self.types
+        @types ||= Set.new %w(
+          binary bits boolean decimal64 empty enumeration int8 int16 int32
+          int64 string uint8 uint16 uint32 uint64 union leafref identityref
+          instance-identifier
+        )
+      end
+
+      state :comment do
+        rule %r/[^*\/]/, Comment
+        rule %r/\/\*/, Comment, :comment
+        rule %r/\*\//, Comment, :pop!
+        rule %r/[*\/]/, Comment
+      end
+
+      #Keyword::Reserved
+      #groups Name::Tag, Text::Whitespace
+      state :root do
+        rule %r/\s+/, Text::Whitespace
+        rule %r/[\{\}\;]+/, Punctuation
+        rule %r/(?<![\-\w])(and|or|not|\+|\.)(?![\-\w])/, Operator
+
+        rule %r/"(?:\\"|[^"])*?"/, Str::Double #for double quotes
+        rule %r/'(?:\\'|[^'])*?'/, Str::Single #for single quotes
+
+        rule %r/\/\*/, Comment, :comment
+        rule %r/\/\/.*?$/, Comment
+
+        #match BNF stmt for `node-identifier` with [ prefix ":"]
+        rule %r/(?:^|(?<=[\s{};]))([\w.-]+)(:)([\w.-]+)(?=[\s{};])/ do
+          groups Name::Namespace, Punctuation, Name
+        end
+
+        #match BNF stmt `date-arg-str`
+        rule %r/([0-9]{4}\-[0-9]{2}\-[0-9]{2})(?=[\s\{\}\;])/, Name::Label
+        rule %r/([0-9]+\.[0-9]+)(?=[\s\{\}\;])/, Num::Float
+        rule %r/([0-9]+)(?=[\s\{\}\;])/, Num::Integer
+
+        rule id do |m|
+          name = m[0].downcase
+
+          if self.class.top_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.module_header_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.meta_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.linkage_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.body_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.data_def_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.type_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.list_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.types.include? name
+            token Keyword::Type
+          elsif self.class.constants_keywords.include? name
+            token Name::Constant
+          else
+            token Name
+          end
+        end
+
+        rule %r/[^;{}\s*+'"]+/, Name
+      end
+    end
+  end
+end

spec/lexers/yang_spec.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+describe Rouge::Lexers::YANG do
+  let(:subject) { Rouge::Lexers::YANG.new }
+
+  describe 'guessing' do
+    include Support::Guessing
+
+    it 'guesses by filename' do
+      assert_guess :filename => 'test.yang'
+    end
+
+    it 'guesses by mimetype' do
+      assert_guess :mimetype => 'application/yang'
+    end
+
+    describe 'lexing' do
+      include Support::Lexing
+
+      it 'parse namespace uri' do
+        assert_tokens_equal 'namespace urn:ietf:params:xml:ns:yang:ietf-alarms;',
+          ['Keyword.Declaration', 'namespace'],
+          ['Text.Whitespace', ' '],
+          ['Name', 'urn:ietf:params:xml:ns:yang:ietf-alarms'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse namespace prefix' do
+        assert_tokens_equal 'type yang:counter64;',
+          ['Keyword.Declaration', 'type'],
+          ['Text.Whitespace', ' '],
+          ['Name.Namespace', 'yang'],
+          ['Punctuation', ":"],
+          ['Name', 'counter64'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse revision-date' do
+        assert_tokens_equal 'revision 2020-03-08{',
+          ['Keyword.Declaration', 'revision'],
+          ['Text.Whitespace', ' '],
+          ['Name.Label', '2020-03-08'],
+          ['Punctuation', '{']
+      end
+
+      it 'parse float in yang-version' do
+        assert_tokens_equal 'yang-version 1.1;',
+          ['Keyword.Declaration', 'yang-version'],
+          ['Text.Whitespace', ' '],
+          ['Literal.Number.Float', '1.1'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse integer in value' do
+        assert_tokens_equal 'value 5;',
+          ['Keyword.Declaration', 'value'],
+          ['Text.Whitespace', ' '],
+          ['Literal.Number.Integer', '5'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse integer in value' do
+        assert_tokens_equal 'value "5";',
+          ['Keyword.Declaration', 'value'],
+          ['Text.Whitespace', ' '],
+          ['Literal.String.Double', '"5"'],
+          ['Punctuation', ';']
+      end
+
+    end
+
+  end
+end

spec/visual/samples/yang

@@ -0,0 +1,64 @@
+module server-system {
+    yang-version 1.1;
+    namespace "http://gribok.org/ns/yang/example";
+    prefix ex;
+
+    import ietf-yang-types {
+      prefix yang;
+      reference
+        "RFC 6991: Common YANG Data Types.";
+    }
+
+    organization "Gribok";
+    contact "[email protected]";
+    description
+        "An example module";
+
+    revision 2020-04-03 {
+        description "Example yang";
+    }
+
+    /*
+     * Comment for container system
+     */
+
+    container system {
+        leaf host-name {
+            type string;
+            description "Hostname for this system";
+        }
+
+        leaf-list domain-search {
+            type string;
+            description "List of domain names to search";
+        }
+
+        container login {
+            leaf message {
+                type string;
+                description
+                    "Message given at start of login session";
+            }
+
+            list user {
+                key "name";
+                leaf name {
+                    type string;
+                }
+                leaf uuid {
+                  type yang:uuid;
+                }
+                leaf full-name {
+                    type string;
+                    mandatory true;
+                    description
+                      "The full name of user  See also 'name'.  This could
+                       be, for example, a reference to the user name";
+                }
+                leaf class {
+                    type string;
+                }
+            }
+        }
+    }
+}