-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Issue #5048 - Review temporary buffer usage #5091
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
// | ||
// ======================================================================== | ||
// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others. | ||
// ------------------------------------------------------------------------ | ||
// All rights reserved. This program and the accompanying materials | ||
// are made available under the terms of the Eclipse Public License v1.0 | ||
// and Apache License v2.0 which accompanies this distribution. | ||
// | ||
// The Eclipse Public License is available at | ||
// http://www.eclipse.org/legal/epl-v10.html | ||
// | ||
// The Apache License v2.0 is available at | ||
// http://www.opensource.org/licenses/apache2.0.php | ||
// | ||
// You may elect to redistribute this code under either of these licenses. | ||
// ======================================================================== | ||
// | ||
|
||
package org.eclipse.jetty.io; | ||
|
||
import java.io.IOException; | ||
import java.io.InputStream; | ||
import java.nio.ByteBuffer; | ||
|
||
import org.eclipse.jetty.util.BufferUtil; | ||
import org.eclipse.jetty.util.resource.Resource; | ||
|
||
public class ByteBufferPoolUtil | ||
{ | ||
public static ByteBuffer resourceToBuffer(Resource resource, boolean direct, ByteBufferPool bufferPool) throws IOException | ||
{ | ||
long len = resource.length(); | ||
if (len < 0) | ||
throw new IllegalArgumentException("invalid resource: " + resource + " len=" + len); | ||
|
||
if (len > Integer.MAX_VALUE) | ||
{ | ||
// This method cannot handle resources of this size. | ||
return null; | ||
} | ||
|
||
int ilen = (int)len; | ||
ByteBuffer buffer; | ||
if (bufferPool != null) | ||
buffer = bufferPool.acquire(ilen, direct); | ||
else | ||
buffer = direct ? BufferUtil.allocateDirect(ilen) : BufferUtil.allocate(ilen); | ||
|
||
int pos = BufferUtil.flipToFill(buffer); | ||
if (resource.getFile() != null) | ||
BufferUtil.readFrom(resource.getFile(), buffer); | ||
else | ||
{ | ||
try (InputStream is = resource.getInputStream()) | ||
{ | ||
BufferUtil.readFrom(is, ilen, buffer); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is the temporary buffers like the one used inside There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also I think we can use our context to use bigger temp buffers inside |
||
} | ||
} | ||
BufferUtil.flipToFlush(buffer, pos); | ||
|
||
return buffer; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -43,6 +43,8 @@ | |
import org.eclipse.jetty.http.PreEncodedHttpField; | ||
import org.eclipse.jetty.http.PrecompressedHttpContent; | ||
import org.eclipse.jetty.http.ResourceHttpContent; | ||
import org.eclipse.jetty.io.ByteBufferPool; | ||
import org.eclipse.jetty.io.ByteBufferPoolUtil; | ||
import org.eclipse.jetty.util.BufferUtil; | ||
import org.eclipse.jetty.util.log.Log; | ||
import org.eclipse.jetty.util.log.Logger; | ||
|
@@ -63,6 +65,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory | |
private final boolean _etags; | ||
private final CompressedContentFormat[] _precompressedFormats; | ||
private final boolean _useFileMappedBuffer; | ||
private final ByteBufferPool _bufferPool; | ||
|
||
private int _maxCachedFileSize = 128 * 1024 * 1024; | ||
private int _maxCachedFiles = 2048; | ||
|
@@ -79,6 +82,22 @@ public class CachedContentFactory implements HttpContent.ContentFactory | |
* @param precompressedFormats array of precompression formats to support | ||
*/ | ||
public CachedContentFactory(CachedContentFactory parent, ResourceFactory factory, MimeTypes mimeTypes, boolean useFileMappedBuffer, boolean etags, CompressedContentFormat[] precompressedFormats) | ||
{ | ||
this(parent, factory, mimeTypes, useFileMappedBuffer, etags, precompressedFormats, null); | ||
} | ||
|
||
/** | ||
* Constructor. | ||
* | ||
* @param parent the parent resource cache | ||
* @param factory the resource factory | ||
* @param mimeTypes Mimetype to use for meta data | ||
* @param useFileMappedBuffer true to file memory mapped buffers | ||
* @param etags true to support etags | ||
* @param precompressedFormats array of precompression formats to support | ||
* @param bufferPool the ByteBufferPool to use | ||
*/ | ||
public CachedContentFactory(CachedContentFactory parent, ResourceFactory factory, MimeTypes mimeTypes, boolean useFileMappedBuffer, boolean etags, CompressedContentFormat[] precompressedFormats, ByteBufferPool bufferPool) | ||
{ | ||
_factory = factory; | ||
_cache = new ConcurrentHashMap<>(); | ||
|
@@ -89,6 +108,7 @@ public CachedContentFactory(CachedContentFactory parent, ResourceFactory factory | |
_useFileMappedBuffer = useFileMappedBuffer; | ||
_etags = etags; | ||
_precompressedFormats = precompressedFormats; | ||
_bufferPool = bufferPool; | ||
} | ||
|
||
public int getCachedSize() | ||
|
@@ -193,9 +213,7 @@ public HttpContent getContent(String pathInContext, int maxBufferSize) throws IO | |
// Is the content in the parent cache? | ||
if (_parent != null) | ||
{ | ||
HttpContent httpContent = _parent.getContent(pathInContext, maxBufferSize); | ||
if (httpContent != null) | ||
return httpContent; | ||
return _parent.getContent(pathInContext, maxBufferSize); | ||
} | ||
|
||
return null; | ||
|
@@ -222,7 +240,7 @@ private HttpContent load(String pathInContext, Resource resource, int maxBufferS | |
return null; | ||
|
||
if (resource.isDirectory()) | ||
return new ResourceHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), getMaxCachedFileSize()); | ||
return new ResourceHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), getMaxCachedFileSize(), null, _bufferPool); | ||
|
||
// Will it fit in the cache? | ||
if (isCacheable(resource)) | ||
|
@@ -237,7 +255,7 @@ private HttpContent load(String pathInContext, Resource resource, int maxBufferS | |
{ | ||
String compressedPathInContext = pathInContext + format._extension; | ||
CachedHttpContent compressedContent = _cache.get(compressedPathInContext); | ||
if (compressedContent == null || compressedContent.isValid()) | ||
if (compressedContent == null || !compressedContent.isValid()) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why was this changed? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We only want to get the resource from the factory again if it is null or not valid. |
||
{ | ||
compressedContent = null; | ||
Resource compressedResource = _factory.getResource(compressedPathInContext); | ||
|
@@ -290,13 +308,13 @@ private HttpContent load(String pathInContext, Resource resource, int maxBufferS | |
if (compressedResource.exists() && compressedResource.lastModified() >= resource.lastModified() && | ||
compressedResource.length() < resource.length()) | ||
compressedContents.put(format, | ||
new ResourceHttpContent(compressedResource, _mimeTypes.getMimeByExtension(compressedPathInContext), maxBufferSize)); | ||
new ResourceHttpContent(compressedResource, _mimeTypes.getMimeByExtension(compressedPathInContext), maxBufferSize, null, _bufferPool)); | ||
} | ||
if (!compressedContents.isEmpty()) | ||
return new ResourceHttpContent(resource, mt, maxBufferSize, compressedContents); | ||
return new ResourceHttpContent(resource, mt, maxBufferSize, compressedContents, _bufferPool); | ||
} | ||
|
||
return new ResourceHttpContent(resource, mt, maxBufferSize); | ||
return new ResourceHttpContent(resource, mt, maxBufferSize, null, _bufferPool); | ||
} | ||
|
||
private void shrinkCache() | ||
|
@@ -335,7 +353,7 @@ protected ByteBuffer getIndirectBuffer(Resource resource) | |
{ | ||
try | ||
{ | ||
return BufferUtil.toBuffer(resource, false); | ||
return ByteBufferPoolUtil.resourceToBuffer(resource, false, _bufferPool); | ||
} | ||
catch (IOException | IllegalArgumentException e) | ||
{ | ||
|
@@ -366,7 +384,7 @@ protected ByteBuffer getDirectBuffer(Resource resource) | |
{ | ||
try | ||
{ | ||
return BufferUtil.toBuffer(resource, true); | ||
return ByteBufferPoolUtil.resourceToBuffer(resource, true, _bufferPool); | ||
} | ||
catch (IOException | IllegalArgumentException e) | ||
{ | ||
|
@@ -485,6 +503,8 @@ boolean isValid() | |
|
||
protected void invalidate() | ||
{ | ||
release(); | ||
|
||
ByteBuffer indirect = _indirectBuffer.getAndSet(null); | ||
if (indirect != null) | ||
_cachedSize.addAndGet(-BufferUtil.length(indirect)); | ||
|
@@ -550,6 +570,16 @@ public Type getMimeType() | |
@Override | ||
public void release() | ||
{ | ||
if (_bufferPool != null) | ||
{ | ||
ByteBuffer indirectBuffer = _indirectBuffer.get(); | ||
if (indirectBuffer != null) | ||
_bufferPool.release(indirectBuffer); | ||
|
||
ByteBuffer directBuffer = _directBuffer.get(); | ||
if (directBuffer != null) | ||
_bufferPool.release(directBuffer); | ||
} | ||
} | ||
|
||
@Override | ||
|
@@ -579,10 +609,11 @@ public ByteBuffer getIndirectBuffer() | |
} | ||
else | ||
{ | ||
_bufferPool.release(buffer2); | ||
buffer = _indirectBuffer.get(); | ||
} | ||
} | ||
return buffer == null ? null : buffer.asReadOnlyBuffer(); | ||
return buffer.slice(); | ||
} | ||
|
||
@Override | ||
|
@@ -615,6 +646,7 @@ else if (_resource.length() < _maxCachedFileSize) | |
} | ||
else | ||
{ | ||
_bufferPool.release(direct); | ||
buffer = _directBuffer.get(); | ||
} | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What is the lifecycle of a
ResourceHttpContent
? Can it be put into a cache? If so, couldgetIndirectBuffer
be called again and again, and the list of allocated buffers get longer and longer with release never called or called in the distant future? Is release always called?I'd be perhaps more included to have a
ByteBuffer getIndirectBuffer(ByteBufferPool pool)
method that the caller can pass in their pool and then be responsible for doing the release themselves.