Merge branch 'mp/reporting-updates'

dev/Vagrantfile

@@ -59,16 +59,14 @@ def define_chef_server(config, attributes)
              "oc_id['db_pool_size']" => 10,
              "oc_bifrost['db_pool_size']" => 10 }
       # TODO merge with other options nicely.
-      json["provisioning"] = {}
-      json["provisioning"]["chef-server-config"] = pg
+      json = simple_deep_merge(json, { "provisioning" => { "chef-server-config" => pg } })
     end
     config.vm.synced_folder File.absolute_path(File.join(Dir.pwd, "../")), "/host",
       type: "rsync",
       rsync__args: ["--verbose", "--archive", "--delete", "-z", "--no-owner", "--no-group" ],
       rsync__exclude: attributes["vm"]["sync-exclude"]
     # We're also going to do a share of the slower vboxsf style, allowing us to auto-checkout dependencies
     # and have them be properly synced to a place that we can use them.
-    config.vm.synced_folder base_path, "/mnt/host-do-not-use"
     config.vm.synced_folder installer_path, "/installers"
     config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig"
     config.vm.provision "shell", inline: install_hack(installer)
@@ -89,7 +87,6 @@ end
 def define_db_server(config, attributes)
   config.vm.hostname = "database.chef-server.dev"
   config.vm.network "private_network", ip: DB_VM_ADDRESS
-
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id,
                   "--name", "database",

dev/cookbooks/dev/templates/default/motd.erb

@@ -1,21 +1,20 @@
 DVM QuickStart
 ---------------
 1. On the host, cd to chef-server/dev, and run the below command. Leave it running.
-      $> ./sync
+      $ ./sync
 2. In your active shell (this one!) use "sudo -i" to gain root. dvm needs it.
-3. To load the chef-server repo's erchef project and run it:
-      #> dvm quickstart oc_erchef
+3. To load the chef-server repo's erchef project and run it in the active shell:
+      # dvm quickstart oc_erchef
+      4. To load the things you'll usually want - erchef, pedant, omnibus cookbooks:
+      # dvm quickstart chef-server
 4. To run oc-chef-pedant, the server REST API spec suite:
-      #> dvm run oc-chef-pedant [--pedant-flags,...]
+      # dvm load oc-chef-pedant
+      # dvm run oc-chef-pedant [--pedant-flags,...]
 5. To start a psql session with a project's database, use:
-      #> dvm psql $PROJECT
+      # dvm psql $PROJECT
 
-The ominbus projects 'private-chef-cookbooks', 'upgrades', and 'ctl-commands' are by
-default loaded in from your chef-server repo clone.  You can instead have chef-server
-instead see the packaged/default versions by unloading them:
-      #> dvm unload omnibus $PROJECT_NAME
-To run the packaged oc-chef-pedant instead of the one from the repo, use:
-      #> cd /opt/opscode/embedded/service/oc-chef-pedant
-      #> bin/oc-chef-pedant -c /var/opt/opscode/oc-chef-pedant/etc/pedant_config.rb [--flags,...]
 Use dvm list for a list of loadable projects, and dvm list $PROJECT for a list of loadable deps.
 For a lot more information about dvm see chef-server/dev/README.md
+
+Note that supported erlang projects will automatically hot-load your changes -
+just save them from the host and they'll loaded within a few seconds.

dev/cookbooks/dev/templates/default/sudoers

@@ -1,4 +1,5 @@
-<% # We do not use env_keep - sudo then lets you share the /vagrant .bashrc and home settings  %>
+Defaults    env_keep+=SSH_AUTH_SOCK
+
 opscode ALL=(ALL:ALL) ALL
 root ALL=(ALL:ALL) ALL
 vagrant ALL=(ALL) NOPASSWD:ALL

dev/defaults.yml

@@ -25,6 +25,7 @@ vm:
   sync-exclude:
     - pkg/
     - deps/
+    - rel/
     - _rel/
     - _build/
     - chef-mover/rel/mover/
@@ -49,6 +50,7 @@ vm:
     - rspec.failures
     - VERSION
     - partybus/config.rb
+    - oc-reporting-pedant/Gemfile.lock # friggin ugh
 
   cover:
     base_output_path: /vagrant/testdata/cover # maps to dev/testdata/cover
@@ -60,7 +62,7 @@ projects:
     type: erlang
     database: opscode_chef # Used for dvm psql oc_erchef
     service:
-      rel-type: relx
+      rel-type: rebar3
       name: opscode-erchef
       cookie: erchef
       node: [email protected]
@@ -70,24 +72,36 @@ projects:
       name: oc_id
   oc_bifrost:
     type: erlang
-    database: opscode_bifrost
+    database: bifrost
     service:
-      rel-type: relx
+      rel-type: rebar3
       name:  oc_bifrost
       cookie: oc_bifrost
       node: [email protected]
+  oc_reporting:
+    type:  erlang
+    external: true
+    make-target: compile generate rel
+    database: opscode_reporting
+    omnibus-project: opscode-reporting
+    service:
+     rel-type: reltool
+     name: opscode-reporting
+     rel-name: oc_reporting
+     cookie: oc_reporting
+     node: [email protected]
   chef-mover:
     type: erlang
     service:
+      rel-type: rebar3
       rel-name: mover
-      rel-type: relx
       name:  opscode-chef-mover
       cookie: mover
       node: [email protected]
   bookshelf:
     type: erlang
     service:
-      rel-type: relx
+      rel-type: rebar3
       name: bookshelf
       cookie: bookshelf
       node: [email protected]
@@ -119,23 +133,66 @@ projects:
         dest_path: /opt/opscode/embedded/service/partybus
         reconfigure_on_load: false
         bundler: true
-  omnibus-ctl:
-    type: ruby #todo instead could callt his 'gem'?
-    system: true
-    external: true
   oc-chef-pedant:
     type: ruby
     path: oc-chef-pedant
     database: opscode_chef
     run: bin/oc-chef-pedant -c /var/opt/opscode/oc-chef-pedant/etc/pedant_config.rb
 
+  #
+  # External Components
+  # We'll eventually allow external projects to supply their own
+  # 'dvm.yml' that we can include for project definitions, but for now...
+
+  #
+  # omnibus-ctl
+  #
+  omnibus-ctl:
+    type: ruby
+    system: true
+    external: true
+
+  #
+  # Reporting
+  #
+  omnibus-reporting:
+    name: omnibus-reporting
+    type: omnibus
+    external: true
+    components:
+      reporting-cookbooks:
+        source_path: cookbooks/opscode-reporting
+        dest_path: /opt/opscode-reporting/embedded/cookbooks/opscode-reporting
+        ctl-name: opscode-reporting-ctl
+        reconfigure_on_load: true
+      ctl-commands:
+        source_path: opscode-reporting-ctl-commands
+        dest_path: /opt/opscode-reporting/embedded/service/omnibus-ctl
+        reconfigure_on_load: false
+  oc-reporting-pedant:
+    type: ruby
+    external: true
+    gem-path: /opt/opscode-reporting/embedded/service/gems
+    run: ./oc-reporting-pedant -c /etc/opscode-reporting/pedant_config.rb
+
 quickstart:
+  chef-server:
+    description: "Load up all the things used for most chef server dev: pedant, cookbooks, commands, erchef.  Starts oc_erchef without a console."
+    load:
+      - omnibus private-chef-cookbooks
+      - omnibus ctl-commands
+      - oc-chef-pedant
+      - oc_erchef
+    start:
+      - oc_erchef
+
   oc_erchef:
     description: "Load up erchef and start it in console mode."
     load:
       - oc_erchef
     start:
       - oc_erchef
+
   oc_bifrost:
     description: "Load oc_bifrost and oc-bifrost-pedant from /host. Start oc_bifrost into a console."
     load:

dev/dvm/lib/dvm/project/erlang.rb

@@ -6,23 +6,36 @@ class ErlangProject < Project
     def initialize(project_name, config)
       super
       @rebar_config_path = "#{project_dir}/rebar.config"
-      reldir = "_build/dev/rel"
-      @relname = service['rel-name'] ? service['rel-name'] : name
+      reldir = case service['rel-type']
+               when 'relx'
+                 "_rel"
+               when 'reltool'
+                 "rel"
+               when 'rebar3'
+                 "_build/dev/rel"
+               end
+
+      @relname = service['rel-name'].nil? ? name : service['rel-name']
       @relpath = "#{@project_dir}/#{reldir}/#{relname}"
-      @service_dir = "/var/opt/opscode/embedded/service/#{service['name']}"
+      @service_dir = "/var/opt/#{omnibus_project}/embedded/service/#{service['name']}"
       @libpath =  File.join(@relpath, "lib")
       @node = service['node']
     end
+    def init_reltool
+
+    end
+    def init_rebar3
+
+    end
 
     def parse_deps
       return @deps if @deps
       @deps = {}
       path = File.expand_path("../../../../parse.es", __FILE__)
-      # For some reason vim tag browser doesn't like it when
-      # eval(`...`) is used.
       result = run_command("#{path} #{@rebar_config_path}")
+      target_lib_dir = service['rel-type'] == 'rebar3' ? "_build/dev/lib" : "deps"
       eval(result.stdout).each do |name, data|
-        @deps[name] = ErlangDep.new(name, File.join(project_dir, "deps") , data, self)
+        @deps[name] = ErlangDep.new(name, File.join(project_dir, target_lib_dir) , data, self)
       end
     end
 
@@ -32,7 +45,7 @@ def is_running?
     end
 
     def start(args, background)
-      raise DVMArgumentError, "#{name} appears to be already running. Try 'dvm console oc_erchef', or 'dvm stop oc_erchef'" if is_running?
+      raise DVMArgumentError, "#{name} appears to be already running. Try 'dvm console #{relname}', or 'dvm stop #{relname}'" if is_running?
       # Ensure that our packaged installation isn't running, thereby preventing spewage of startup errors.
       disable_service
       if background
@@ -42,7 +55,7 @@ def start(args, background)
       end
     end
 
-    def stop()
+    def stop
       raise DVMArgumentError, <<-EOM unless is_running?
 "#{name} does not seem to be running from a loaded instance.
 If you want to stop the global instance, use chef-server-ctl stop #{service['name']}'"
@@ -88,7 +101,7 @@ def loaded?
     def link
       say("Setting up symlinks")
       # Yay project inconsistencies
-      base_sv_path = "/var/opt/opscode/#{service["name"]}"
+      base_sv_path = "/var/opt/#{omnibus_project}/#{service["name"]}"
       purge_links
       if File.exists?("#{base_sv_path}/sys.config")
         FileUtils.ln_s("#{base_sv_path}/sys.config", "#{relpath}/sys.config")
@@ -104,13 +117,9 @@ def purge_links
     end
 
     def do_load(options)
-      # TODO: Can we just link in our dev-time dependencies post-build,
-      # so that we don't have to declare them as apps in app.src? These are
-      # typically sync, builder, eunit , mixer and common_test
-      #
       # TODO this can also be wrapped and handled in the base...
-      # TODO this makes an assumption that this is NOT anything bundled in our chef-server env.
-      if ! project_dir_exists_on_host?(path)
+      puts "Checking #{path}"
+      if !project_dir_exists_on_host?(path)
         git = project['git']
         if git
           if git['uri']
@@ -120,7 +129,8 @@ def do_load(options)
             end
           end
         else
-          raise DVMArgumentError, "Project not available. Clone it onto your host machine, or update the git settings in config.yml."
+          # UGH _ note we're still stuck on not having /do-not-use - because if we clone to external-deps it will not get nuked!
+          raise DVMArgumentError, "Project not available. Clone or link it into chef-server/external-deps directory onto your host machine, or update the project git settings."
         end
       end
 
@@ -165,7 +175,8 @@ def do_build
       # TODO currently we only load projects that are maintained by Chef and live in chef-server/src.
       # If this changes, we'll need to support specification of alternative build commands.
       # TODO2: add build.env since only erchef needs use_system_gecode...
-      run_command("make -j 4 dvm", "Building...", cwd: project_dir, env: { "USE_SYSTEM_GECODE" => "1"})
+      make_target = project['make-target'] ? project['make-target'] : 'dvm'
+      run_command("make -j 4 #{make_target}", "Building...", cwd: project_dir, env: { "USE_SYSTEM_GECODE" => "1"})
     end
 
     def cover(action, modulename, options)

dev/dvm/lib/dvm/project/erlang_dep.rb

@@ -43,7 +43,7 @@ def load_info
       @available = true
       @libpath = matches[0]
       @libname = File.basename(matches[0])
-      @link_target = File.readlink(libpath)
+      @link_target = File.symlink?(libpath) ? File.readlink(libpath) : libpath
     end
 
 
@@ -57,12 +57,12 @@ def load(opts)
         if opts[:skip_checkout]
           raise DVMArgumentError "The project directory for #{name} can't be found, and you told me not to check it out."
         end
-        clone(name, url)
         # Ensure we're starting with the same code base that we had in the dependency to avoid
         # hot-loading headaches.
+        clone(name, url)
       end
-      # TODO verify project is running, or allow option to do build anyway (just not  y default?)
-      # Problem being if the project is not running, we will not auto-compiel
+      # TODO if project is not running, do a normal build of the project - or shoudl we anyway so that
+      #      our beam files persist?
 
       checkout(name, ref) unless opts[:skip_checkout]
       # INstead of linking it into the dep directory, replace the library path in the project installation

dev/dvm/lib/dvm/project/omnibus.rb

@@ -17,7 +17,7 @@ def parse_deps()
     def loaded?
       File.directory? @project_dir
     end
-    def do_load
+    def do_load(ignore)
       # TODO we could actually add the standard clone-if-missing support
       say "This project is not loadable on its own. Instead use 'dvm load #{name} $subproject"
     end

dev/dvm/lib/dvm/project/omnibus_dep.rb

@@ -1,6 +1,6 @@
 module DVM
   class OmnibusDep < Dep
-    attr_reader :source_path, :dest_path, :reconfigure_on_load, :bundler
+    attr_reader :source_path, :dest_path, :reconfigure_on_load, :bundler, :ctl_name
     # Needed for now to make tools mixin behave:
     attr_reader :path
     def initialize(base_dir, name, config)
@@ -10,12 +10,13 @@ def initialize(base_dir, name, config)
       @path = @source_path
       @reconfigure_on_load = config['reconfigure_on_load']
       @available = true
+      @ctl_name  = config['ctl-name'] || 'chef-server-ctl'
       @bundler = config['bundler']
     end
     def unload
       unmount(source_path)
       if reconfigure_on_load
-        run_command("chef-server-ctl reconfigure", "Reconfiguring chef server to pick up the changes")
+        run_command("#{ctl_name} reconfigure", "running #{ctl_name} reconfigure to up the changes.")
       end
     end
     def load(opts)
@@ -24,7 +25,7 @@ def load(opts)
       end
       bind_mount(source_path, dest_path)
       if reconfigure_on_load and not opts[:no_build]
-        run_command("chef-server-ctl reconfigure", "Reconfiguring chef server to pick up the changes")
+        run_command("#{ctl_name} reconfigure", "running #{ctl_name} reconfigure to up the changes.")
       end
 
     end

dev/dvm/lib/dvm/project/project.rb

@@ -1,19 +1,24 @@
 require "mixlib/shellout"
 module DVM
   class Project
-    attr_reader :name, :project, :config, :service, :project_dir, :path, :external
+    attr_reader :name, :project, :config, :service, :project_dir, :path, :external, :omnibus_project
     include DVM::Tools
     # TODO check required fields in config
     def initialize(project_name, config)
       @project = config['projects'][project_name]
       @external = @project['external'] || false
+      # We're not doing auto-checkout for now - so tell them they need to find a mising thing
+      @omnibus_project = @project['omnibus-project'] || "opscode"
       @name = project.has_key?('name') ? project['name'] : project_name
       if external
         @path = project['path'] || "external-deps/#{name}"
       else
         @path = project['path'] || "src/#{name}"
       end
       @project_dir = "/host/#{path}"
+      if (external and !File.exists?(@project_dir))
+        raise DVM::DVMArgumentError, "Please check out or symlink #{name} into chef-server/external-deps on your host before attempting to load it."
+      end
       @config = config
       @service = @project['service']
     end

dev/dvm/lib/dvm/project/ruby.rb

@@ -3,9 +3,14 @@ module DVM
   # be overlaid into the /opt/opscode/embedded/lib/ruby/gems/X/$project directory
   # via bind mount.  This will ensure it's available for use in chef-server-ctl commands
   # which aren't run from in the dvm substitute environment.
+  # TODO: support for 'system' in another system - like omnibus-reporting env...
   class RubyProject < Project
+    attr_reader :gem_path, :with_binstubs
     def initialize(project_name, config)
       super
+      @gem_path = project['gem-path'] || "/opt/ospcode/embedded/service/gem"
+      @with_binstubs = project.has_key?('with-binstubs') ? project['with-binstubs'] : false
+
     end
     def do_load(options)
       if @project['system']
@@ -24,7 +29,9 @@ def load_ruby_project
       # so that they can just be run via `dvm run #{name}` without having to mess around
       # with the gems installed in package ruby.
       run_command("rm -rf .bundle/config", cwd: project_dir)
-      run_command("bundle install --path /opt/opscode/embedded/service/gem --no-binstubs", "Installing in-place...", cwd: project_dir)
+      binstubs = with_binstubs ? "--with-binstubs" : "--no-binstubs"
+      path = gem_path == "none" ? "" : "--path #{gem_path}"
+      run_command("bundle install #{path} #{binstubs}", "Installing in-place...", cwd: project_dir)
     end
     def unload
       unmount(@project_dir)

dev/dvm/lib/dvm/tools.rb

@@ -55,27 +55,29 @@ def system_gem_path(name)
        Pathname.new(gem_path).parent.parent.to_s
     end
     def host_raw_dir
-        "/mnt/host-do-not-use"
+        "/host"
     end
 
     def host_project_dir(path)
       File.join(host_raw_dir, path)
     end
 
     def clone(name, uri)
-      run_command("git clone '#{uri}' '#{name}'", "Cloning #{name} to host.", cwd: host_raw_dir)
+      run_command("git clone '#{uri}' '#{name}'", "Cloning #{name} to host. For future reference, you may also symlink it into chef-server/external-deps from another location on the host.",
+                  cwd: host_external_deps_dir)
     end
 
     def project_dir_exists_on_host?(name)
-      File.directory? host_project_dir(name)
+      puts "Checking #{host_project_dir(name)} "
+      File.directory?(host_project_dir(name))
     end
 
     def checkout(name, ref)
       cwd = host_project_dir(name)
       result = run_command("git checkout #{ref}", "Checking out #{ref} to match what is currently running", cwd: cwd, no_raise: true)
       if result.error?
         raise DVM::DVMArgumentError, <<-EOM
-Could not check out #{ref} in #{host_project_dir(name)}.
+Could not check out #{ref} in #{cwd}.
 Have you pulled the latest and/or stashed local changes?
 Alternatively, specify '--no-checkout'.