Add YANG lexer

lib/rouge/demos/yang

@@ -0,0 +1,52 @@
+module test-module {
+  namespace "urn:ietf:params:xml:ns:yang:test";
+  prefix tmodule;
+
+  import yang-types {
+    prefix yang;
+  }
+
+  organization "My Organization";
+  contact
+    "My name is Max Mustermann";
+  description
+    "This is an example description";
+
+  revision 2020-03-08 {
+    description
+      "Initial revision.";
+  }
+
+  typedef address-family {
+    type enumeration {
+      enum ipv4 {
+        value 1;
+        description
+          "IPv4 Address Family.";
+      }
+
+      enum ipv6 {
+        value 2;
+        description
+          "IPv6 Address Family.";
+      }
+    }
+  }
+
+  list interface-list {
+      key "name";
+      description
+        "This represents the list of interfaces";
+      leaf name {
+        type yang:interface-type;
+        description
+          "A reference to the interface type definition.";
+      }
+    }
+
+  feature myfeature {
+    description
+      "Example for a feature";
+    reference "No reference exists";
+  }
+}

lib/rouge/lexers/yang.rb

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class YANG < RegexLexer
+      title 'YANG'
+      desc "Lexer for the YANG 1.1 modeling language (RFC7950)"
+      tag 'yang'
+      filenames '*.yang'
+      mimetypes 'application/yang'
+
+      id = /[\w_-]+(?=[^\w\-\:])\b/
+
+      #Keywords from RFC7950 ; oriented at BNF style
+      def self.top_stmts_keywords
+        @top_stms_keywords ||= Set.new %w(
+          module submodule
+        )
+      end
+
+      def self.module_header_stmts_keywords
+        @module_header_stmts_keywords ||= Set.new %w(
+          yang-version namespace prefix belongs-to
+        )
+      end
+
+      def self.meta_stmts_keywords
+        @meta_stmts_keywords ||= Set.new %w(
+          organization contact description reference revision
+        )
+      end
+
+      def self.linkage_stmts_keywords
+        @linkage_stmts_keywords ||= Set.new %w(
+          import include revision-date
+        )
+      end
+
+      def self.body_stmts_keywords
+        @body_stms_keywords ||= Set.new %w(
+          extension feature identity typedef grouping augment rpc notification
+          deviation action argument identity if-feature input output
+        )
+      end
+
+      def self.data_def_stmts_keywords
+        @data_def_stms_keywords ||= Set.new %w(
+          container leaf-list leaf list choice anydata anyxml uses case config
+          deviate must when presence refine
+        )
+      end
+
+      def self.type_stmts_keywords
+        @type_stmts_keywords ||= Set.new %w(
+          type units default status bit enum error-app-tag error-message
+          fraction-digits length min-elements max-elements modifier ordered-by
+          path pattern position range require-instance value yin-element base
+        )
+      end
+
+      def self.list_stmts_keywords
+        @list_stmts_keywords ||= Set.new %w(
+          key mandatory unique
+        )
+      end
+
+      #RFC7950 other keywords
+      def self.constants_keywords
+        @constants_keywords ||= Set.new %w(
+          true false current obsolete deprecated add delete replace
+          not-supported invert-match max min unbounded user
+        )
+      end
+
+      #RFC7950 Built-In Types
+      def self.types
+        @types ||= Set.new %w(
+          binary bits boolean decimal64 empty enumeration int8 int16 int32
+          int64 string uint8 uint16 uint32 uint64 union leafref identityref
+          instance-identifier
+        )
+      end
+
+      state :comment do
+        rule %r/[^*\/]/, Comment
+        rule %r/\/\*/, Comment, :comment
+        rule %r/\*\//, Comment, :pop!
+        rule %r/[*\/]/, Comment
+      end
+
+      #Keyword::Reserved
+      #groups Name::Tag, Text::Whitespace
+      state :root do
+        rule %r/\s+/, Text::Whitespace
+        rule %r/[\{\}\;]+/, Punctuation
+        rule %r/(?<![\-\w])(and|or|not|\+|\.)(?![\-\w])/, Operator
+
+        rule %r/"(?:\\"|[^"])*?"/, Str::Double #for double quotes
+        rule %r/'(?:\\'|[^'])*?'/, Str::Single #for single quotes
+
+        rule %r/\/\*/, Comment, :comment
+        rule %r/\/\/.*?$/, Comment
+
+        #match BNF stmt for `node-identifier` with [ prefix ":"]
+        rule %r/(?:^|(?<=[\s\{\}\;]))([^;{}\s\*\+\'\"\:\/]+)(:)([^;{}\s\*\+\'\"\:\/]+)(?=[\s\{\}\;])/ do
+          groups Name::Namespace, Punctuation, Name
+        end
+
+        #match BNF stmt `date-arg-str`
+        rule %r/([0-9]{4}\-[0-9]{2}\-[0-9]{2})(?=[\s\{\}\;])/, Name::Label
+        rule %r/([0-9]+\.[0-9]+)(?=[\s\{\}\;])/, Num::Float
+        rule %r/([0-9]+)(?=[\s\{\}\;])/, Num::Integer
+
+        rule id do |m|
+          name = m[0].downcase
+
+          if self.class.top_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.module_header_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.meta_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.linkage_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.body_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.data_def_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.type_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.list_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.types.include? name
+            token Keyword::Type
+          elsif self.class.constants_keywords.include? name
+            token Name::Constant
+          else
+            token Name
+          end
+        end
+
+        rule %r/[^;{}\s*+'"]+/, Name
+      end
+    end
+  end
+end

spec/lexers/yang_spec.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+describe Rouge::Lexers::YANG do
+  let(:subject) { Rouge::Lexers::YANG.new }
+
+  describe 'guessing' do
+    include Support::Guessing
+
+    it 'guesses by filename' do
+      assert_guess :filename => 'test.yang'
+    end
+
+    it 'guesses by mimetype' do
+      assert_guess :mimetype => 'application/yang'
+    end
+
+    describe 'lexing' do
+      include Support::Lexing
+
+      it 'parse namespace uri' do
+        assert_tokens_equal 'namespace urn:ietf:params:xml:ns:yang:ietf-alarms;',
+          ['Keyword.Declaration', 'namespace'],
+          ['Text.Whitespace', ' '],
+          ['Name', 'urn:ietf:params:xml:ns:yang:ietf-alarms'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse namespace prefix' do
+        assert_tokens_equal 'type yang:counter64;',
+          ['Keyword.Declaration', 'type'],
+          ['Text.Whitespace', ' '],
+          ['Name.Namespace', 'yang'],
+          ['Punctuation', ":"],
+          ['Name', 'counter64'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse revision-date' do
+        assert_tokens_equal 'revision 2020-03-08{',
+          ['Keyword.Declaration', 'revision'],
+          ['Text.Whitespace', ' '],
+          ['Name.Label', '2020-03-08'],
+          ['Punctuation', '{']
+      end
+
+      it 'parse float in yang-version' do
+        assert_tokens_equal 'yang-version 1.1;',
+          ['Keyword.Declaration', 'yang-version'],
+          ['Text.Whitespace', ' '],
+          ['Literal.Number.Float', '1.1'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse integer in value' do
+        assert_tokens_equal 'value 5;',
+          ['Keyword.Declaration', 'value'],
+          ['Text.Whitespace', ' '],
+          ['Literal.Number.Integer', '5'],
+          ['Punctuation', ';']
+      end
+
+      it 'parse integer in value' do
+        assert_tokens_equal 'value "5";',
+          ['Keyword.Declaration', 'value'],
+          ['Text.Whitespace', ' '],
+          ['Literal.String.Double', '"5"'],
+          ['Punctuation', ';']
+      end
+
+    end
+
+  end
+end