api: add follow param to file stream endpoint (#6049)

The `/v1/client/fs/stream endpoint` supports tailing a file by writing
chunks out as they come in. But not all browsers support streams
(ex IE11) so we need to be able to tail a file without streaming.

The fs stream and logs endpoint use the same implementation for
filesystem streaming under the hood, but the fs stream always passes
the `follow` parameter set to true. This adds the same toggle to the
fs stream endpoint that we have for logs. It defaults to true for
backwards compatibility.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 IMPROVEMENTS:
  * agent: allow the job GC interval to be configured [[GH-5978](https://github.com/hashicorp/nomad/issues/5978)]
+ * api: add follow parameter to file streaming endpoint to support older browsers [[GH-6049](https://github.com/hashicorp/nomad/issues/6049)]
 
 ## 0.9.5 (Unreleased)
 
@@ -46,7 +47,7 @@ BUG FIXES:
  * driver: Fixed an issue preventing external driver plugins from launching executor process [[GH-5726](https://github.com/hashicorp/nomad/issues/5726)]
  * driver/docker: Fixed a bug mounting relative paths on Windows [[GH-5811](https://github.com/hashicorp/nomad/issues/5811)]
  * driver/exec: Upgraded libcontainer dependency to avoid zombie `runc:[1:CHILD]]` processes [[GH-5851](https://github.com/hashicorp/nomad/issues/5851)]
- * metrics: Added metrics for raft and state store indexes. [[GH-5841](https://github.com/hashicorp/nomad/issues/5841)] 
+ * metrics: Added metrics for raft and state store indexes. [[GH-5841](https://github.com/hashicorp/nomad/issues/5841)]
  * metrics: Upgrade prometheus client to avoid label conflicts [[GH-5850](https://github.com/hashicorp/nomad/issues/5850)]
  * ui: Fixed ability to click sort arrow to change sort direction [[GH-5833](https://github.com/hashicorp/nomad/pull/5833)]
 
@@ -1629,4 +1630,3 @@ BUG FIXES:
 ## 0.1.0 (September 28, 2015)
 
   * Initial release
-

command/agent/fs_endpoint.go

@@ -194,11 +194,13 @@ func (s *HTTPServer) FileCatRequest(resp http.ResponseWriter, req *http.Request)
 // Stream streams the content of a file blocking on EOF.
 // The parameters are:
 // * path: path to file to stream.
+// * follow: A boolean of whether to follow the file, defaults to true.
 // * offset: The offset to start streaming data at, defaults to zero.
 // * origin: Either "start" or "end" and defines from where the offset is
 //           applied. Defaults to "start".
 func (s *HTTPServer) Stream(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	var allocID, path string
+	var err error
 
 	q := req.URL.Query()
 
@@ -210,10 +212,16 @@ func (s *HTTPServer) Stream(resp http.ResponseWriter, req *http.Request) (interf
 		return nil, fileNameNotPresentErr
 	}
 
+	follow := true
+	if followStr := q.Get("follow"); followStr != "" {
+		if follow, err = strconv.ParseBool(followStr); err != nil {
+			return nil, fmt.Errorf("failed to parse follow field to boolean: %v", err)
+		}
+	}
+
 	var offset int64
 	offsetString := q.Get("offset")
 	if offsetString != "" {
-		var err error
 		if offset, err = strconv.ParseInt(offsetString, 10, 64); err != nil {
 			return nil, fmt.Errorf("error parsing offset: %v", err)
 		}
@@ -234,7 +242,7 @@ func (s *HTTPServer) Stream(resp http.ResponseWriter, req *http.Request) (interf
 		Path:    path,
 		Origin:  origin,
 		Offset:  offset,
-		Follow:  true,
+		Follow:  follow,
 	}
 	s.parse(resp, req, &fsReq.QueryOptions.Region, &fsReq.QueryOptions)
 
@@ -265,13 +273,13 @@ func (s *HTTPServer) Logs(resp http.ResponseWriter, req *http.Request) (interfac
 
 	if followStr := q.Get("follow"); followStr != "" {
 		if follow, err = strconv.ParseBool(followStr); err != nil {
-			return nil, fmt.Errorf("Failed to parse follow field to boolean: %v", err)
+			return nil, fmt.Errorf("failed to parse follow field to boolean: %v", err)
 		}
 	}
 
 	if plainStr := q.Get("plain"); plainStr != "" {
 		if plain, err = strconv.ParseBool(plainStr); err != nil {
-			return nil, fmt.Errorf("Failed to parse plain field to boolean: %v", err)
+			return nil, fmt.Errorf("failed to parse plain field to boolean: %v", err)
 		}
 	}
 

command/agent/fs_endpoint_test.go

@@ -341,7 +341,54 @@ func TestHTTP_FS_Cat(t *testing.T) {
 	})
 }
 
-func TestHTTP_FS_Stream(t *testing.T) {
+func TestHTTP_FS_Stream_NoFollow(t *testing.T) {
+	t.Parallel()
+	require := require.New(t)
+	httpTest(t, nil, func(s *TestAgent) {
+		a := mockFSAlloc(s.client.NodeID(), nil)
+		addAllocToClient(s, a, terminalClientAlloc)
+
+		offset := 4
+		expectation := base64.StdEncoding.EncodeToString(
+			[]byte(defaultLoggerMockDriverStdout[len(defaultLoggerMockDriverStdout)-offset:]))
+		path := fmt.Sprintf("/v1/client/fs/stream/%s?path=alloc/logs/web.stdout.0&offset=%d&origin=end&follow=false",
+			a.ID, offset)
+
+		p, _ := io.Pipe()
+		req, err := http.NewRequest("GET", path, p)
+		require.Nil(err)
+		respW := testutil.NewResponseRecorder()
+		doneCh := make(chan struct{})
+		go func() {
+			_, err = s.Server.Stream(respW, req)
+			require.Nil(err)
+			close(doneCh)
+		}()
+
+		out := ""
+		testutil.WaitForResult(func() (bool, error) {
+			output, err := ioutil.ReadAll(respW)
+			if err != nil {
+				return false, err
+			}
+
+			out += string(output)
+			return strings.Contains(out, expectation), fmt.Errorf("%q doesn't contain %q", out, expectation)
+		}, func(err error) {
+			t.Fatal(err)
+		})
+
+		select {
+		case <-doneCh:
+		case <-time.After(1 * time.Second):
+			t.Fatal("should close but did not")
+		}
+
+		p.Close()
+	})
+}
+
+func TestHTTP_FS_Stream_Follow(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)
 	httpTest(t, nil, func(s *TestAgent) {

website/source/api/client.html.md

@@ -9,7 +9,7 @@ description: |-
 
 # Client HTTP API
 
-The `/client` endpoints are used to interact with the Nomad clients. 
+The `/client` endpoints are used to interact with the Nomad clients.
 
 Since Nomad 0.8.0, both a client and server can handle client endpoints. This is
 particularly useful for when a direct connection to a client is not possible due
@@ -363,6 +363,8 @@ The table below shows this endpoint's support for
 - `path` `(string: "/")` - Specifies the path of the file to read, relative to
   the root of the allocation directory.
 
+- `follow` `(bool: true)`- Specifies whether to tail the file.
+
 - `offset` `(int: <required>)` - Specifies the byte offset from where content
   will be read.