V1 controller with agent

dev-tools/packaging/packages.yml

@@ -398,6 +398,14 @@ shared:
         source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/elastic-agent-shipper-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+        source: '{{.AgentDropPath}}/elastic-agent-shipper-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/elastic-agent-shipper-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{.AgentDropPath}}/elastic-agent-shipper-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+        skip_on_missing: true
       'data/{{.BeatName}}-{{ commit_short }}/downloads/cloudbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{.AgentDropPath}}/cloudbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
@@ -669,12 +677,49 @@ shared:
       source: _meta/kibana.generated
       mode: 0644
 
+  - &binary_component_files
+    '{{.BeatName}}{{.BinaryExt}}':
+      source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+      mode: 0755
+    LICENSE.txt:
+      source: '{{ repo.RootDir }}/LICENSE.txt'
+      mode: 0644
+    NOTICE.txt:
+      source: '{{ repo.RootDir }}/NOTICE.txt'
+      mode: 0644
+    .build_hash.txt:
+      content: >
+        {{ commit }}
+      mode: 0644
+    '{{.BeatName}}.yml':
+      source: '{{.BeatName}}.yml'
+      mode: 0600
+      config: true
+
   # Binary package spec (tar.gz for linux/darwin) for community beats.
   - &binary_spec
     <<: *common
     files:
       <<: *binary_files
 
+    # Binary spec for "components" such as the shipper
+  - &binary_component_spec
+    <<: *common
+    files:
+      <<: *binary_component_files
+
+  # Binary package spec (zip for windows) for community beats.
+  - &windows_binary_component_spec
+    <<: *common
+    files:
+      <<: *binary_component_files
+      install-service-{{.BeatName}}.ps1:
+        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/install-service.ps1.tmpl'
+        mode: 0755
+      uninstall-service-{{.BeatName}}.ps1:
+        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
+        mode: 0755
+
   # Binary package spec (zip for windows) for community beats.
   - &windows_binary_spec
     <<: *common
@@ -917,6 +962,24 @@ specs:
         <<: *binary_spec
         <<: *elastic_license_for_binaries
 
+  #elastic-agent-shipper packaged for the agent
+  elastic_shipper_agent:
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *windows_binary_component_spec
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *binary_component_spec
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *binary_component_spec
+
+
   # Elastic Beat with Elastic License and binary taken the current directory.
   elastic_beat_xpack_reduced:
     ###

internal/spec/elastic-agent-shipper.spec.yml

@@ -0,0 +1,13 @@
+name: Shipper
+cmd: elastic-agent-shipper
+args: []
+artifact: elastic-agent-shipper
+restart_on_output_change: true
+
+rules:
+  - fix_stream: {}
+  - filter:
+      selectors:
+        - shipper
+
+when:  length(${shipper}) > 0

magefile.go

@@ -92,6 +92,15 @@ type Demo mg.Namespace
 // Dev runs package and build for dev purposes.
 type Dev mg.Namespace
 
+// this maps pre-packaged binaries to if they're x-pack beats
+// see packageAgent for more context
+var packedBeats = map[string]bool{
+	"filebeat":    true,
+	"heartbeat":   true,
+	"metricbeat":  true,
+	"osquerybeat": true,
+}
+
 // Notice regenerates the NOTICE.txt file.
 func Notice() error {
 	fmt.Println(">> Generating NOTICE")
@@ -676,10 +685,10 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 		defer os.RemoveAll(dropPath)
 		defer os.Unsetenv(agentDropPath)
 
-		packedBeats := []string{"filebeat", "heartbeat", "metricbeat", "osquerybeat"}
+		//packedBeats := []string{"filebeat", "heartbeat", "metricbeat", "osquerybeat"}
 		if devtools.ExternalBuild == true {
 			ctx := context.Background()
-			for _, beat := range packedBeats {
+			for beat, _ := range packedBeats {
 				for _, reqPackage := range requiredPackages {
 					newVersion, packageName := getPackageName(beat, version, reqPackage)
 					err := fetchBinaryFromArtifactsApi(ctx, packageName, beat, newVersion, dropPath)
@@ -689,14 +698,24 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 				}
 			}
 		} else {
+			// This shouldn't be needed as a special append() here, but until our artiface APIs have released shipper packages,
+			// we need to make sure the package command doesn't try to download a shipper tarball and blow up
+			packedBeats["elastic-agent-shipper"] = false
 			// build from local repo, will assume beats repo is located on the same root level
-			for _, b := range packedBeats {
-				pwd, err := filepath.Abs(filepath.Join("../beats/x-pack", b))
+			fmt.Println(">>> Building from local repo")
+			for beat, isXpack := range packedBeats {
+				var pwd string
+				var err error
+				if !isXpack {
+					pwd, err = filepath.Abs(filepath.Join("../", beat))
+				} else {
+					pwd, err = filepath.Abs(filepath.Join("../beats/x-pack", beat))
+				}
 				if err != nil {
 					panic(err)
 				}
 
-				if !requiredPackagesPresent(pwd, b, version, requiredPackages) {
+				if !requiredPackagesPresent(pwd, beat, version, requiredPackages) {
 					cmd := exec.Command("mage", "package")
 					cmd.Dir = pwd
 					cmd.Stdout = os.Stdout
@@ -729,6 +748,7 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 }
 
 func fetchBinaryFromArtifactsApi(ctx context.Context, packageName, artifact, version, downloadPath string) error {
+	fmt.Printf(">>> About to download: %s\n", packageName)
 	location, err := downloads.FetchBeatsBinary(
 		ctx,
 		packageName,