[IPO-202] Add initial Data Collector application and /_status check

.gitignore

@@ -10,3 +10,4 @@ opt/
 NEW_CHANGELOG.md
 version-manifest.json
 MODIFIED_COMPONENTS_CHANGELOG.md
+/dev/nodes

omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb

@@ -801,3 +801,30 @@
 default['private_chef']['folsom_graphite']['send_interval'] = 10000
 # if a connection fails, how frequently do we attempt to reconnect?
 default['private_chef']['folsom_graphite']['retry_interval'] = 2000
+
+#
+# Data Collector
+#
+# data_collector configuration for erchef. These are used to configure an
+# opscoderl_httpc pool of HTTP connecton workers.
+# If a root_url and token are present the erchef will start the data_collector
+# application.
+
+# Fully qualified URL to the data collector server (e.g.: https://localhost/insights).
+# default['private_chef']['data_collector']['root_url']
+# The authentication token to pass via the header to the data collector server
+# default['private_chef']['data_collector']['token']
+# Timeout for requests to the data collector server in milliseconds.
+default['private_chef']['data_collector']['timeout'] = 30000
+# How many HTTP workers to start in the pool.
+default['private_chef']['data_collector']['http_init_count'] = 25
+# Maximum number of HTTP workers in the pool.
+default['private_chef']['data_collector']['http_max_count'] = 100
+# Maximum age of a server pool worker before terminating it.
+default['private_chef']['data_collector']['http_max_age'] = "{70, sec}"
+# How often to cull aged-out connections.
+default['private_chef']['data_collector']['http_cull_interval'] = "{1, min}"
+# Maximum age of a connection before terminating it.
+default['private_chef']['data_collector']['http_max_connection_duration'] = "{70,sec}"
+# Options for the ibrowse connections (see ibrowse).
+default['private_chef']['data_collector']['ibrowse_options'] = "[{connect_timeout, 10000}]"

omnibus/files/private-chef-cookbooks/private-chef/libraries/private_chef.rb

@@ -89,6 +89,8 @@ module PrivateChef
   backup Mash.new
   backup["strategy"] = "tar"
 
+  data_collector Mash.new
+
   # - legacy config mashes -
   # these config values are here so that if any config has been previously
   # set for these projects in an older version of private-chef/chef-server.rb
@@ -215,6 +217,7 @@ def generate_hash
         "enabled_plugins",
         "license",
         "backup",
+        "data_collector",
 
         # keys for cleanup and back-compat
         "couchdb",
@@ -223,13 +226,16 @@ def generate_hash
       (default_keys | keys_from_extensions).each do |key|
         # @todo: Just pick a naming convention and adhere to it
         # consistently
-        rkey = if key =~ /^oc_/ || key == "redis_lb" ||
-                  key == "use_chef_backend" ||
-                  key == "chef_backend_members"
-                 key # leave oc_* keys as is
-               else
-                 key.gsub("_", "-")
-               end
+        rkey = if key =~ /^oc_/ || %w{
+          redis_lb
+          use_chef_backend
+          chef_backend_members
+          data_collector
+        }.include?(key)
+          key
+        else
+          key.gsub("_", "-")
+        end
         results["private_chef"][rkey] = PrivateChef[key]
       end
       results["private_chef"]["default_orgname"] = PrivateChef["default_orgname"]

omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb

@@ -1,6 +1,6 @@
 %% -*- mode: erlang -*-
 %% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
-%% ex: ts=4 sw=4 ft=erlang et
+%% ex: ts=4 sw=4 ft=eruby.erlang et
 [
     {kernel, [{inet_dist_use_interface, {127,0,0,1}}]},
     %% SASL config
@@ -111,7 +111,14 @@
         {max_request_size, <%= node['private_chef']['opscode-erchef']['max_request_size'] %>},
         {server_version, "<%= node['private_chef']['version'] %>"},
         {health_ping_timeout, <%= node['private_chef']['opscode-erchef']['health_ping_timeout'] %>},
-        {health_ping_modules, [oc_chef_authz, chef_sql, chef_<%= node['private_chef']['opscode-erchef']['search_provider'] %>]},
+        {health_ping_modules, [
+            <% if node['private_chef']['data_collector']['root_url'] %>
+            data_collector,
+            <% end %>
+            oc_chef_authz,
+            chef_sql,
+            chef_<%= node['private_chef']['opscode-erchef']['search_provider'] %>
+        ]},
         {base_resource_url, <%= @helper.erl_atom_or_string(node['private_chef']['opscode-erchef']['base_resource_url']) %>},
         {bulk_fetch_batch_size, <%= @bulk_fetch_batch_size %>},
         {strict_search_result_acls, <%= @strict_search_result_acls %>},
@@ -242,6 +249,20 @@
         {depsolver_pooler_timeout, <%= @depsolver_pooler_timeout %>}
     ]},
 
+    <% if node['private_chef']['data_collector']['root_url'] %>
+    {data_collector, [
+        {root_url,                "<%= node['private_chef']['data_collector']['root_url'] %>"},
+        {token,                   "<%= node['private_chef']['data_collector']['token'] %>"},
+        {timeout,                 <%= node['private_chef']['data_collector']['timeout'] %>},
+        {init_count,              <%= node['private_chef']['data_collector']['http_init_count'] %>},
+        {max_count,               <%= node['private_chef']['data_collector']['http_max_count'] %>},
+        {cull_interval,           <%= node['private_chef']['data_collector']['http_cull_interval'] %>},
+        {max_age,                 <%= node['private_chef']['data_collector']['http_max_age'] %>},
+        {max_connection_duration, <%= node['private_chef']['data_collector']['http_max_connection_duration'] %>},
+        {ibrowse_options,         <%= node['private_chef']['data_collector']['ibrowse_options'] %>}
+    ]},
+    <% end %>
+
     {stats_hero, [
         %% Set sender pool size to DB max_connections to avoid contention
         {udp_socket_pool_size, <%= node['private_chef']['postgresql']['max_connections'] %> },

src/oc_erchef/apps/data_collector/.gitignore

@@ -0,0 +1,15 @@
+.eunit
+deps/
+ebin/
+ebin_dialyzer/
+TAGS
+.DS_Store
+doc/*.html
+*.beam
+/doc/edoc-info
+/doc/erlang.png
+/doc/stylesheet.css
+/deps.plt
+test/*.out
+.rebar
+log/

src/oc_erchef/apps/data_collector/Makefile

@@ -0,0 +1,30 @@
+REBAR3_URL=https://s3.amazonaws.com/rebar3/rebar3
+
+# If there is a rebar in the current directory, use it
+ifeq ($(wildcard rebar3),rebar3)
+REBAR3 = $(CURDIR)/rebar3
+endif
+
+# Fallback to rebar on PATH
+REBAR3 ?= $(shell which rebar3)
+
+# And finally, prep to download rebar if all else fails
+ifeq ($(REBAR3),)
+REBAR3 = rebar3
+endif
+
+all: $(REBAR3)
+	@$(REBAR3) do clean, compile, eunit, dialyzer
+
+rel: all
+	@$(REBAR3) release
+
+distclean:
+	@rm -rf _build
+
+$(REBAR3):
+	curl -Lo rebar3 $(REBAR3_URL) || wget $(REBAR3_URL)
+	chmod a+x rebar3
+
+install: $(REBAR3) distclean
+	$(REBAR3) update

src/oc_erchef/apps/data_collector/README.md

@@ -0,0 +1,3 @@
+# Data Collector
+
+Data Collector is an HTTP exporter of Chef Server actions for external services.

src/oc_erchef/apps/data_collector/rebar.config

@@ -0,0 +1,23 @@
+%% -*- mode: erlang -*-
+%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+{profiles, [{
+    test, [
+        {deps, [meck]},
+        {erl_opts, [export_all]}
+    ]
+}]}.
+
+{erl_opts, [
+    warnings_as_errors,
+    {parse_transform, lager_transform},
+    debug_info
+]}.
+
+{deps, [
+    {lager, ".*", {git, "https://github.com/basho/lager", {tag, "2.1.1"}}},
+    {opscoderl_httpc, ".*", {git, "https://github.com/chef/opscoderl_httpc", {branch, "master"}}},
+    {pooler, ".*", {git,"git://github.com/seth/pooler.git", {branch,"master"}}}
+]}.
+
+{cover_enabled, true}.

src/oc_erchef/apps/data_collector/rebar.lock

@@ -0,0 +1,12 @@
+[{<<"ibrowse">>,
+  {git,"git://github.com/opscode/ibrowse.git",
+       {ref,"8f3f6a3a30730b193cc340a8885a960586dc98de"}},
+  1},
+ {<<"opscoderl_httpc">>,
+  {git,"https://github.com/chef/opscoderl_httpc",
+       {ref,"41a0cb853f4273be394eb952c9eb4eae2d3f00d4"}},
+  0},
+ {<<"pooler">>,
+  {git,"git://github.com/seth/pooler.git",
+       {ref,"521f568bf9a2ccbe7c7e0fc23f24cd06ec559b79"}},
+  0}].

src/oc_erchef/apps/data_collector/src/data_collector.app.src

@@ -0,0 +1,35 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*-
+%% ex: ts=4 sw=4 et
+%%
+%% @author Ryan Cragun <[email protected]>
+%% @author John Keiser <[email protected].
+%%
+%% Copyright 2016 Chef Software, Inc. All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+
+{application, data_collector, [
+    {description, "Chef Server Data Collector"},
+    {vsn, {cmd,"cat ../../VERSION | awk '{print $0}'"}},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib,
+        lager,
+        opscoderl_httpc
+    ]},
+    {mod, {data_collector_app, []}}
+]}.

src/oc_erchef/apps/data_collector/src/data_collector.erl

@@ -0,0 +1,52 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*-
+%% ex: ts=4 sw=4 et
+%%
+%% @author Ryan Cragun <[email protected]>
+%% @author John Keiser <[email protected].
+%%
+%% Copyright 2016 Chef Software, Inc. All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+
+-module(data_collector).
+-export([
+         ping/0,
+         update/1,
+         is_enabled/0
+        ]).
+
+-spec ping() -> pong | pang.
+ping() ->
+    case data_collector_http:get("/") of
+        ok -> pong;
+        _ -> pang
+    end.
+
+-spec update(iolist() | binary()) -> ok | {error, term()}.
+update(Body) when is_list(Body) ->
+    update(iolist_to_binary(Body));
+update(Body) ->
+    %% TODO: Transform to data collector JSON
+    data_collector_http:post("/", Body).
+
+-spec is_enabled() -> boolean().
+is_enabled() ->
+    case application:get_env(data_collector, root_url) of
+        {ok, _Value} ->
+            true;
+        undefined ->
+            false
+    end.

src/oc_erchef/apps/data_collector/src/data_collector_app.erl

@@ -0,0 +1,36 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*-
+%% ex: ts=4 sw=4 et
+%% @author Ryan Cragun <[email protected]>
+%% @author John Keiser <[email protected].
+%%
+%% Copyright 2016 Chef Software, Inc. All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+
+-module(data_collector_app).
+
+-behaviour(application).
+
+%% API
+-export([start/2,
+         stop/1
+        ]).
+
+start(_StartType, _StartArgs) ->
+    data_collector_sup:start_link().
+
+stop(_State) ->
+    ok.