Skip to content

Commit

Permalink
(dev/core#174) Memcache(d) - Updates to comply with PSR-16
Browse files Browse the repository at this point in the history 
There are two drivers, `CRM_Utils_Memcache` and `CRM_Utils_Memcached`.  It's
nice to update them in tandem (with similar design decisions). If an admin
admin is experimenting/debugging, this consistency

In addition to the standard PSR-16-style changes, there are a couple changes
in how data is formatted when written to memcache:

* To allow support for targetted `flush()`ing (one prefix at a time), we update
  the naming convention per https://github.com/memcached/memcached/wiki/ProgrammingTricks#deleting-by-namespace
  This means that a typical key includes a bucket-revision code:
    * BEFORE: `<site-prefix>/<bucket-prefix>/<item-key>` (`dmaster/default/mykey`)
    * BEFORE: `<site-prefix>/<bucket-prefix>/<bucket-revision>/<item-key>` (`dmaster/default/5b33011fea555/mykey`)
* Values are `serialize()`d. This resolves an ambiguity where `Memcache::get()`
  does not let us know if it returns `FALSE` because there's an error because
  that's the stored value. By serializing, those scenarios can be distinguished.
    * `get(...) === FALSE` means "item was not found"
    * `get(...) === serialize(FALSE)` means "item was found with value FALSE"
  • Loading branch information
@totten
totten committed Jun 27, 2018
1 parent 0295bce commit 975a7fe
Show file tree
Hide file tree
Showing 2 changed files with 157 additions and 33 deletions.
72 changes: 57 additions & 15 deletions CRM/Utils/Cache/Memcache.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,17 @@
class CRM_Utils_Cache_Memcache implements CRM_Utils_Cache_Interface {

use CRM_Utils_Cache_NaiveMultipleTrait; // TODO Consider native implementation.
use CRM_Utils_Cache_NaiveHasTrait; // TODO Native implementation

const DEFAULT_HOST = 'localhost';
const DEFAULT_PORT = 11211;
const DEFAULT_TIMEOUT = 3600;
const DEFAULT_PREFIX = '';

/**
* If another process clears namespace, we'll find out in ~5 sec.
*/
const NS_LOCAL_TTL = 5;

/**
* The host name of the memcached server.
*
Expand Down Expand Up @@ -79,6 +83,15 @@ class CRM_Utils_Cache_Memcache implements CRM_Utils_Cache_Interface {
*/
protected $_cache;

/**
* @var NULL|array
*
* This is the effective prefix. It may be bumped up whenever the dataset is flushed.
*
* @see https://github.com/memcached/memcached/wiki/ProgrammingTricks#deleting-by-namespace
*/
protected $_truePrefix = NULL;

/**
* Constructor.
*
Expand Down Expand Up @@ -118,13 +131,12 @@ public function __construct($config) {
* @return bool
*/
public function set($key, $value, $ttl = NULL) {
if ($ttl !== NULL) {
throw new \RuntimeException("FIXME: " . __CLASS__ . "::set() should support non-NULL TTL");
}
if (!$this->_cache->set($this->_prefix . $key, $value, FALSE, $this->_timeout)) {
return FALSE;
CRM_Utils_Cache::assertValidKey($key);
if (is_int($ttl) && $ttl <= 0) {
return $this->delete($key);
}
return TRUE;
$expires = CRM_Utils_Date::convertCacheTtlToExpires($ttl, $this->_timeout);
return $this->_cache->set($this->getTruePrefix() . $key, serialize($value), FALSE, $expires);
}

/**
Expand All @@ -134,32 +146,62 @@ public function set($key, $value, $ttl = NULL) {
* @return mixed
*/
public function get($key, $default = NULL) {
if ($default !== NULL) {
throw new \RuntimeException("FIXME: " . __CLASS__ . "::get() only supports NULL default");
}
$result = $this->_cache->get($this->_prefix . $key);
return $result;
CRM_Utils_Cache::assertValidKey($key);
$result = $this->_cache->get($this->getTruePrefix() . $key);
return ($result === FALSE) ? $default : unserialize($result);
}

/**
* @param string $key
*
* @return bool
* @throws \Psr\SimpleCache\CacheException
*/
public function has($key) {
CRM_Utils_Cache::assertValidKey($key);
$result = $this->_cache->get($this->getTruePrefix() . $key);
return ($result !== FALSE);
}


/**
* @param $key
*
* @return bool
*/
public function delete($key) {
return $this->_cache->delete($this->_prefix . $key);
CRM_Utils_Cache::assertValidKey($key);
$this->_cache->delete($this->getTruePrefix() . $key);
return TRUE;
}

/**
* @return bool
*/
public function flush() {
// FIXME: Only delete items matching `$this->_prefix`.
return $this->_cache->flush();
$this->_truePrefix = NULL;
$this->_cache->delete($this->_prefix);
return TRUE;
}

public function clear() {
return $this->flush();
}

protected function getTruePrefix() {
if ($this->_truePrefix === NULL || $this->_truePrefix['expires'] < time()) {
$key = $this->_prefix;
$value = $this->_cache->get($key);
if ($value === FALSE) {
$value = uniqid();
$this->_cache->set($key, $value, FALSE, 0); // Indefinite.
}
$this->_truePrefix = [
'value' => $value,
'expires' => time() + self::NS_LOCAL_TTL,
];
}
return $this->_prefix . $this->_truePrefix['value'] . '/';
}

}
118 changes: 100 additions & 18 deletions CRM/Utils/Cache/Memcached.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,17 @@
class CRM_Utils_Cache_Memcached implements CRM_Utils_Cache_Interface {

use CRM_Utils_Cache_NaiveMultipleTrait; // TODO Consider native implementation.
use CRM_Utils_Cache_NaiveHasTrait; // TODO Native implementation

const DEFAULT_HOST = 'localhost';
const DEFAULT_PORT = 11211;
const DEFAULT_TIMEOUT = 3600;
const DEFAULT_PREFIX = '';
const MAX_KEY_LEN = 62;
const MAX_KEY_LEN = 200;

/**
* If another process clears namespace, we'll find out in ~5 sec.
*/
const NS_LOCAL_TTL = 5;

/**
* The host name of the memcached server
Expand Down Expand Up @@ -80,6 +84,15 @@ class CRM_Utils_Cache_Memcached implements CRM_Utils_Cache_Interface {
*/
protected $_cache;

/**
* @var NULL|array
*
* This is the effective prefix. It may be bumped up whenever the dataset is flushed.
*
* @see https://github.com/memcached/memcached/wiki/ProgrammingTricks#deleting-by-namespace
*/
protected $_truePrefix = NULL;

/**
* Constructor.
*
Expand Down Expand Up @@ -120,14 +133,23 @@ public function __construct($config) {
* @throws Exception
*/
public function set($key, $value, $ttl = NULL) {
if ($ttl !== NULL) {
throw new \RuntimeException("FIXME: " . __CLASS__ . "::set() should support non-NULL TTL");
CRM_Utils_Cache::assertValidKey($key);
if (is_int($ttl) && $ttl <= 0) {
return $this->delete($key);
}
$expires = CRM_Utils_Date::convertCacheTtlToExpires($ttl, $this->_timeout);

$key = $this->cleanKey($key);
if (!$this->_cache->set($key, $value, $this->_timeout)) {
CRM_Core_Error::debug('Result Code: ', $this->_cache->getResultMessage());
CRM_Core_Error::fatal("memcached set failed, wondering why?, $key", $value);
if (!$this->_cache->set($key, serialize($value), $expires)) {
if (PHP_SAPI === 'cli' || (Civi\Core\Container::isContainerBooted() && CRM_Core_Permission::check('view debug output'))) {
throw new CRM_Utils_Cache_CacheException("Memcached::set($key) failed: " . $this->_cache->getResultMessage());
}
else {
Civi::log()->error("Redis set ($key) failed: " . $this->_cache->getResultMessage());
throw new CRM_Utils_Cache_CacheException("Memcached::set($key) failed");
}
return FALSE;

}
return TRUE;
}
Expand All @@ -139,12 +161,45 @@ public function set($key, $value, $ttl = NULL) {
* @return mixed
*/
public function get($key, $default = NULL) {
if ($default !== NULL) {
throw new \RuntimeException("FIXME: " . __CLASS__ . "::get() only supports NULL default");
}
CRM_Utils_Cache::assertValidKey($key);
$key = $this->cleanKey($key);
$result = $this->_cache->get($key);
return $result;
switch ($this->_cache->getResultCode()) {
case Memcached::RES_SUCCESS:
return unserialize($result);

case Memcached::RES_NOTFOUND:
return $default;

default:
Civi::log()->error("Memcached::get($key) failed: " . $this->_cache->getResultMessage());
throw new CRM_Utils_Cache_CacheException("Memcached set ($key) failed");
}
}

/**
* @param string $key
*
* @return bool
* @throws \Psr\SimpleCache\CacheException
*/
public function has($key) {
CRM_Utils_Cache::assertValidKey($key);
$key = $this->cleanKey($key);
if ($this->_cache->get($key) !== FALSE) {
return TRUE;
}
switch ($this->_cache->getResultCode()) {
case Memcached::RES_NOTFOUND:
return FALSE;

case Memcached::RES_SUCCESS:
return TRUE;

default:
Civi::log()->error("Memcached::has($key) failed: " . $this->_cache->getResultMessage());
throw new CRM_Utils_Cache_CacheException("Memcached set ($key) failed");
}
}

/**
Expand All @@ -153,8 +208,13 @@ public function get($key, $default = NULL) {
* @return mixed
*/
public function delete($key) {
CRM_Utils_Cache::assertValidKey($key);
$key = $this->cleanKey($key);
return $this->_cache->delete($key);
if ($this->_cache->delete($key)) {
return TRUE;
}
$code = $this->_cache->getResultCode();
return ($code == Memcached::RES_DELETED || $code == Memcached::RES_NOTFOUND);
}

/**
Expand All @@ -163,25 +223,47 @@ public function delete($key) {
* @return mixed|string
*/
public function cleanKey($key) {
$key = preg_replace('/\s+|\W+/', '_', $this->_prefix . $key);
if (strlen($key) > self::MAX_KEY_LEN) {
$truePrefix = $this->getTruePrefix();
$maxLen = self::MAX_KEY_LEN - strlen($truePrefix);
$key = preg_replace('/\s+|\W+/', '_', $key);
if (strlen($key) > $maxLen) {
$md5Key = md5($key); // this should be 32 characters in length
$subKeyLen = self::MAX_KEY_LEN - 1 - strlen($md5Key);
$subKeyLen = $maxLen - 1 - strlen($md5Key);
$key = substr($key, 0, $subKeyLen) . "_" . $md5Key;
}
return $key;
return $truePrefix . $key;
}

/**
* @return bool
*/
public function flush() {
// FIXME: Only delete items matching `$this->_prefix`.
return $this->_cache->flush();
$this->_truePrefix = NULL;
if ($this->_cache->delete($this->_prefix)) {
return TRUE;
}
$code = $this->_cache->getResultCode();
return ($code == Memcached::RES_DELETED || $code == Memcached::RES_NOTFOUND);
}

public function clear() {
return $this->flush();
}

protected function getTruePrefix() {
if ($this->_truePrefix === NULL || $this->_truePrefix['expires'] < time()) {
$key = $this->_prefix;
$value = $this->_cache->get($key);
if ($this->_cache->getResultCode() === Memcached::RES_NOTFOUND) {
$value = uniqid();
$this->_cache->add($key, $value, 0); // Indefinite.
}
$this->_truePrefix = [
'value' => $value,
'expires' => time() + self::NS_LOCAL_TTL,
];
}
return $this->_prefix . $this->_truePrefix['value'] . '/';
}

}

0 comments on commit 975a7fe

Please sign in to comment.