Make licensing FIPS-140 compliant (#30251)

Necessary changes so that the licensing functionality can be
used in a JVM in FIPS 140 approved mode.
* Uses adequate salt length in encryption
* Changes key derivation to PBKDF2WithHmacSHA512 from a custom
  approach with SHA512 and manual key stretching
* Removes redundant manual padding

Other relevant changes:
* Uses the SAH512 hash instead of the encrypted key bytes as the
  key fingerprint to be included in the license specification
* Removes the explicit verification check of the encryption key
  as this is implicitly checked in signature verification.

x-pack/license-tools/src/main/java/org/elasticsearch/license/licensor/LicenseSigner.java

@@ -8,6 +8,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -20,7 +21,10 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.InvalidKeyException;
+import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
 import java.security.SecureRandom;
 import java.security.Signature;
 import java.security.SignatureException;
@@ -35,9 +39,7 @@
 public class LicenseSigner {
 
     private static final int MAGIC_LENGTH = 13;
-
     private final Path publicKeyPath;
-
     private final Path privateKeyPath;
 
     public LicenseSigner(final Path privateKeyPath, final Path publicKeyPath) {
@@ -59,9 +61,11 @@ public License sign(License licenseSpec) throws IOException {
                 Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true");
         licenseSpec.toXContent(contentBuilder, new ToXContent.MapParams(licenseSpecViewMode));
         final byte[] signedContent;
+        final boolean preV4 = licenseSpec.version() < License.VERSION_CRYPTO_ALGORITHMS;
         try {
             final Signature rsa = Signature.getInstance("SHA512withRSA");
-            rsa.initSign(CryptUtils.readEncryptedPrivateKey(Files.readAllBytes(privateKeyPath)));
+            PrivateKey decryptedPrivateKey = CryptUtils.readEncryptedPrivateKey(Files.readAllBytes(privateKeyPath));
+            rsa.initSign(decryptedPrivateKey);
             final BytesRefIterator iterator = BytesReference.bytes(contentBuilder).iterator();
             BytesRef ref;
             while((ref = iterator.next()) != null) {
@@ -77,20 +81,28 @@ public License sign(License licenseSpec) throws IOException {
         final byte[] magic = new byte[MAGIC_LENGTH];
         SecureRandom random = new SecureRandom();
         random.nextBytes(magic);
-        final byte[] hash = Base64.getEncoder().encode(Files.readAllBytes(publicKeyPath));
-        assert hash != null;
-        byte[] bytes = new byte[4 + 4 + MAGIC_LENGTH + 4 + hash.length + 4 + signedContent.length];
+        final byte[] publicKeyBytes = Files.readAllBytes(publicKeyPath);
+        PublicKey publicKey = CryptUtils.readPublicKey(publicKeyBytes);
+        final byte[] pubKeyFingerprint = preV4 ? Base64.getEncoder().encode(CryptUtils.writeEncryptedPublicKey(publicKey)) :
+                getPublicKeyFingerprint(publicKeyBytes);
+        byte[] bytes = new byte[4 + 4 + MAGIC_LENGTH + 4 + pubKeyFingerprint.length + 4 + signedContent.length];
         ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
         byteBuffer.putInt(licenseSpec.version())
                 .putInt(magic.length)
                 .put(magic)
-                .putInt(hash.length)
-                .put(hash)
+                .putInt(pubKeyFingerprint.length)
+                .put(pubKeyFingerprint)
                 .putInt(signedContent.length)
                 .put(signedContent);
 
         return License.builder()
                 .fromLicenseSpec(licenseSpec, Base64.getEncoder().encodeToString(bytes))
                 .build();
     }
+
+    private byte[] getPublicKeyFingerprint(byte[] keyBytes) {
+        MessageDigest sha256 = MessageDigests.sha256();
+        sha256.update(keyBytes);
+        return sha256.digest();
+    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/license/CryptUtils.java

@@ -15,95 +15,71 @@
 import javax.crypto.spec.PBEKeySpec;
 import javax.crypto.spec.SecretKeySpec;
 import java.nio.charset.StandardCharsets;
-import java.security.InvalidKeyException;
+import java.security.GeneralSecurityException;
 import java.security.KeyFactory;
 import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.security.PrivateKey;
 import java.security.PublicKey;
 import java.security.SecureRandom;
+import java.security.NoSuchAlgorithmException;
+import java.security.InvalidKeyException;
 import java.security.spec.InvalidKeySpecException;
 import java.security.spec.PKCS8EncodedKeySpec;
 import java.security.spec.X509EncodedKeySpec;
 import java.util.Base64;
 
 public class CryptUtils {
-    private static final int minimumPadding = 20;
-    private static final byte[] salt = {
-            (byte) 0xA9, (byte) 0xA2, (byte) 0xB5, (byte) 0xDE,
-            (byte) 0x2A, (byte) 0x8A, (byte) 0x9A, (byte) 0xE6
+    // SALT must be at least 128bits for FIPS 140-2 compliance
+    private static final byte[] SALT = {
+            (byte) 0x74, (byte) 0x68, (byte) 0x69, (byte) 0x73,
+            (byte) 0x69, (byte) 0x73, (byte) 0x74, (byte) 0x68,
+            (byte) 0x65, (byte) 0x73, (byte) 0x61, (byte) 0x6C,
+            (byte) 0x74, (byte) 0x77, (byte) 0x65, (byte) 0x75
     };
-    private static final int iterationCount = 1024;
-    private static final int aesKeyLength = 128;
-    private static final String keyAlgorithm = "RSA";
-    private static final String passHashAlgorithm = "SHA-512";
-    private static final String DEFAULT_PASS_PHRASE = "elasticsearch-license";
-
-    private static final SecureRandom random = new SecureRandom();
+    private static final String KEY_ALGORITHM = "RSA";
+    private static final char[] DEFAULT_PASS_PHRASE = "elasticsearch-license".toCharArray();
+    private static final String KDF_ALGORITHM = "PBKDF2WithHmacSHA512";
+    private static final int KDF_ITERATION_COUNT = 10000;
+    private static final String CIPHER_ALGORITHM = "AES";
+    // This can be changed to 256 once Java 9 is the minimum version
+    // http://www.oracle.com/technetwork/java/javase/terms/readme/jdk9-readme-3852447.html#jce
+    private static final int ENCRYPTION_KEY_LENGTH = 128;
+    private static final SecureRandom RANDOM = new SecureRandom();
 
     /**
      * Read encrypted private key file content with default pass phrase
      */
     public static PrivateKey readEncryptedPrivateKey(byte[] fileContents) {
-        try {
-            return readEncryptedPrivateKey(fileContents, hashPassPhrase(DEFAULT_PASS_PHRASE));
-        } catch (NoSuchAlgorithmException e) {
-            throw new IllegalStateException(e);
-        }
-    }
-
-    /**
-     * Read encrypted public key file content with default pass phrase
-     */
-    public static PublicKey readEncryptedPublicKey(byte[] fileContents) {
-        try {
-            return readEncryptedPublicKey(fileContents, hashPassPhrase(DEFAULT_PASS_PHRASE));
-        } catch (NoSuchAlgorithmException e) {
-            throw new IllegalStateException(e);
-        }
-    }
-
-    /**
-     * Returns encrypted public key file content with default pass phrase
-     */
-    public static byte[] writeEncryptedPublicKey(PublicKey publicKey) {
-        try {
-            return writeEncryptedPublicKey(publicKey, hashPassPhrase(DEFAULT_PASS_PHRASE));
-        } catch (NoSuchAlgorithmException e) {
-            throw new IllegalStateException(e);
-        }
+        return readEncryptedPrivateKey(fileContents, DEFAULT_PASS_PHRASE, false);
     }
 
     /**
      * Returns encrypted private key file content with default pass phrase
      */
     public static byte[] writeEncryptedPrivateKey(PrivateKey privateKey) {
-        try {
-            return writeEncryptedPrivateKey(privateKey, hashPassPhrase(DEFAULT_PASS_PHRASE));
-        } catch (NoSuchAlgorithmException e) {
-            throw new IllegalStateException(e);
-        }
+        return writeEncryptedPrivateKey(privateKey, DEFAULT_PASS_PHRASE);
     }
 
     /**
      * Read encrypted private key file content with provided <code>passPhrase</code>
      */
-    public static PrivateKey readEncryptedPrivateKey(byte[] fileContents, char[] passPhrase) {
-        PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(decrypt(fileContents, passPhrase));
+    public static PrivateKey readEncryptedPrivateKey(byte[] fileContents, char[] passPhrase, boolean preV4) {
+        byte[] keyBytes = preV4 ? decryptV3Format(fileContents) : decrypt(fileContents, passPhrase);
+        PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(keyBytes);
         try {
-            return KeyFactory.getInstance(keyAlgorithm).generatePrivate(privateKeySpec);
+            return KeyFactory.getInstance(KEY_ALGORITHM).generatePrivate(privateKeySpec);
         } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
             throw new IllegalStateException(e);
         }
     }
 
     /**
-     * Read encrypted public key file content with provided <code>passPhrase</code>
+     * Read public key file content
      */
-    public static PublicKey readEncryptedPublicKey(byte[] fileContents, char[] passPhrase) {
-        X509EncodedKeySpec publicKeySpec = new X509EncodedKeySpec(decrypt(fileContents, passPhrase));
+    public static PublicKey readPublicKey(byte[] fileContents) {
+        X509EncodedKeySpec publicKeySpec = new X509EncodedKeySpec(fileContents);
         try {
-            return KeyFactory.getInstance(CryptUtils.keyAlgorithm).generatePublic(publicKeySpec);
+            return KeyFactory.getInstance(CryptUtils.KEY_ALGORITHM).generatePublic(publicKeySpec);
         } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
             throw new IllegalStateException(e);
         }
@@ -112,9 +88,9 @@ public static PublicKey readEncryptedPublicKey(byte[] fileContents, char[] passP
     /**
      * Returns encrypted public key file content with provided <code>passPhrase</code>
      */
-    public static byte[] writeEncryptedPublicKey(PublicKey publicKey, char[] passPhrase) {
+    public static byte[] writeEncryptedPublicKey(PublicKey publicKey) {
         X509EncodedKeySpec encodedKeySpec = new X509EncodedKeySpec(publicKey.getEncoded());
-        return encrypt(encodedKeySpec.getEncoded(), passPhrase);
+        return encrypt(encodedKeySpec.getEncoded(), DEFAULT_PASS_PHRASE);
     }
 
     /**
@@ -128,33 +104,25 @@ public static byte[] writeEncryptedPrivateKey(PrivateKey privateKey, char[] pass
     /**
      * Encrypts provided <code>data</code> with <code>DEFAULT_PASS_PHRASE</code>
      */
-    public static byte[] encrypt(byte[] data) {
-        try {
-            return encrypt(data, hashPassPhrase(DEFAULT_PASS_PHRASE));
-        } catch (NoSuchAlgorithmException e) {
-            throw new IllegalStateException(e);
-        }
+    static byte[] encrypt(byte[] data) {
+        return encrypt(data, DEFAULT_PASS_PHRASE);
     }
 
     /**
      * Decrypts provided <code>encryptedData</code> with <code>DEFAULT_PASS_PHRASE</code>
      */
-    public static byte[] decrypt(byte[] encryptedData) {
-        try {
-            return decrypt(encryptedData, hashPassPhrase(DEFAULT_PASS_PHRASE));
-        } catch (NoSuchAlgorithmException e) {
-            throw new IllegalStateException(e);
-        }
+    static byte[] decrypt(byte[] encryptedData) {
+        return decrypt(encryptedData, DEFAULT_PASS_PHRASE);
     }
 
     /**
      * Encrypts provided <code>data</code> with <code>passPhrase</code>
      */
-    public static byte[] encrypt(byte[] data, char[] passPhrase) {
+    private static byte[] encrypt(byte[] data, char[] passPhrase) {
         try {
-            final Cipher encryptionCipher = getEncryptionCipher(getSecretKey(passPhrase));
-            return encryptionCipher.doFinal(pad(data, minimumPadding));
-        } catch (InvalidKeySpecException | IllegalBlockSizeException | BadPaddingException e) {
+            final Cipher encryptionCipher = getEncryptionCipher(deriveSecretKey(passPhrase));
+            return encryptionCipher.doFinal(data);
+        } catch (IllegalBlockSizeException | BadPaddingException e) {
             throw new IllegalStateException(e);
         }
     }
@@ -164,29 +132,60 @@ public static byte[] encrypt(byte[] data, char[] passPhrase) {
      */
     private static byte[] decrypt(byte[] encryptedData, char[] passPhrase) {
         try {
-            final Cipher cipher = getDecryptionCipher(getSecretKey(passPhrase));
-            return unPad(cipher.doFinal(encryptedData));
-        } catch (IllegalBlockSizeException | BadPaddingException | InvalidKeySpecException e) {
+            final Cipher cipher = getDecryptionCipher(deriveSecretKey(passPhrase));
+            return cipher.doFinal(encryptedData);
+        } catch (IllegalBlockSizeException | BadPaddingException e) {
             throw new IllegalStateException(e);
         }
+    }
 
+    static byte[] encryptV3Format(byte[] data) {
+        try {
+            SecretKey encryptionKey = getV3Key();
+            final Cipher encryptionCipher = getEncryptionCipher(encryptionKey);
+            return encryptionCipher.doFinal(pad(data, 20));
+        } catch (GeneralSecurityException e) {
+            throw new IllegalStateException(e);
+        }
     }
 
-    private static SecretKey getSecretKey(char[] passPhrase) throws InvalidKeySpecException {
+    static byte[] decryptV3Format(byte[] data) {
         try {
-            PBEKeySpec keySpec = new PBEKeySpec(passPhrase, salt, iterationCount, aesKeyLength);
+            SecretKey decryptionKey = getV3Key();
+            final Cipher decryptionCipher = getDecryptionCipher(decryptionKey);
+            return unPad(decryptionCipher.doFinal(data));
+        } catch (GeneralSecurityException e) {
+            throw new IllegalStateException(e);
+        }
+    }
 
-            byte[] shortKey = SecretKeyFactory.getInstance("PBEWithSHA1AndDESede").
-                    generateSecret(keySpec).getEncoded();
+    private static SecretKey getV3Key() throws NoSuchAlgorithmException, InvalidKeySpecException {
+        final byte[] salt = {
+                (byte) 0xA9, (byte) 0xA2, (byte) 0xB5, (byte) 0xDE,
+                (byte) 0x2A, (byte) 0x8A, (byte) 0x9A, (byte) 0xE6
+        };
+        final byte[] passBytes = "elasticsearch-license".getBytes(StandardCharsets.UTF_8);
+        final byte[] digest = MessageDigest.getInstance("SHA-512").digest(passBytes);
+        final char[] hashedPassphrase = Base64.getEncoder().encodeToString(digest).toCharArray();
+        PBEKeySpec keySpec = new PBEKeySpec(hashedPassphrase, salt, 1024, 128);
+        byte[] shortKey = SecretKeyFactory.getInstance("PBEWithSHA1AndDESede").
+                generateSecret(keySpec).getEncoded();
+        byte[] intermediaryKey = new byte[16];
+        for (int i = 0, j = 0; i < 16; i++) {
+            intermediaryKey[i] = shortKey[j];
+            if (++j == shortKey.length)
+                j = 0;
+        }
+        return new SecretKeySpec(intermediaryKey, "AES");
+    }
 
-            byte[] intermediaryKey = new byte[aesKeyLength / 8];
-            for (int i = 0, j = 0; i < aesKeyLength / 8; i++) {
-                intermediaryKey[i] = shortKey[j];
-                if (++j == shortKey.length)
-                    j = 0;
-            }
+    private static SecretKey deriveSecretKey(char[] passPhrase) {
+        try {
+            PBEKeySpec keySpec = new PBEKeySpec(passPhrase, SALT, KDF_ITERATION_COUNT, ENCRYPTION_KEY_LENGTH);
 
-            return new SecretKeySpec(intermediaryKey, "AES");
+            SecretKey secretKey = SecretKeyFactory.getInstance(KDF_ALGORITHM).
+                    generateSecret(keySpec);
+            return new SecretKeySpec(secretKey.getEncoded(), CIPHER_ALGORITHM);
         } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
             throw new IllegalStateException(e);
         }
@@ -202,8 +201,8 @@ private static Cipher getDecryptionCipher(SecretKey secretKey) {
 
     private static Cipher getCipher(int mode, SecretKey secretKey) {
         try {
-            Cipher cipher = Cipher.getInstance(secretKey.getAlgorithm());
-            cipher.init(mode, secretKey, random);
+            Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM);
+            cipher.init(mode, secretKey, RANDOM);
             return cipher;
         } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchPaddingException e) {
             throw new IllegalStateException(e);
@@ -228,7 +227,7 @@ private static byte[] pad(byte[] bytes, int length) {
 
         // fill the rest with random bytes
         byte[] fill = new byte[padded - 1];
-        random.nextBytes(fill);
+        RANDOM.nextBytes(fill);
         System.arraycopy(fill, 0, out, i, padded - 1);
 
         out[length] = (byte) (padded + 1);
@@ -246,10 +245,4 @@ private static byte[] unPad(byte[] bytes) {
 
         return out;
     }
-
-    private static char[] hashPassPhrase(String passPhrase) throws NoSuchAlgorithmException {
-        final byte[] passBytes = passPhrase.getBytes(StandardCharsets.UTF_8);
-        final byte[] digest = MessageDigest.getInstance(passHashAlgorithm).digest(passBytes);
-        return Base64.getEncoder().encodeToString(digest).toCharArray();
-    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java

@@ -38,7 +38,8 @@ public class License implements ToXContentObject {
     public static final int VERSION_START = 1;
     public static final int VERSION_NO_FEATURE_TYPE = 2;
     public static final int VERSION_START_DATE = 3;
-    public static final int VERSION_CURRENT = VERSION_START_DATE;
+    public static final int VERSION_CRYPTO_ALGORITHMS = 4;
+    public static final int VERSION_CURRENT = VERSION_CRYPTO_ALGORITHMS;
 
     /**
      * XContent param name to deserialize license(s) with

x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java

@@ -402,9 +402,9 @@ public void clusterChanged(ClusterChangedEvent event) {
 
             boolean noLicense = noLicenseInPrevMetadata && noLicenseInCurrentMetadata;
             // auto-generate license if no licenses ever existed or if the current license is basic and
-            // needs extended. this will trigger a subsequent cluster changed event
-            if (currentClusterState.getNodes().isLocalNodeElectedMaster()
-                    && (noLicense || LicenseUtils.licenseNeedsExtended(currentLicense))) {
+            // needs extended or if the license signature needs to be updated. this will trigger a subsequent cluster changed event
+            if (currentClusterState.getNodes().isLocalNodeElectedMaster() &&
+                    (noLicense || LicenseUtils.licenseNeedsExtended(currentLicense) || LicenseUtils.signatureNeedsUpdate(currentLicense))) {
                 registerOrUpdateSelfGeneratedLicense();
             }
         } else if (logger.isDebugEnabled()) {

x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseUtils.java

@@ -37,4 +37,13 @@ public static boolean isLicenseExpiredException(ElasticsearchSecurityException e
     public static boolean licenseNeedsExtended(License license) {
         return "basic".equals(license.type()) && license.expiryDate() != LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS;
     }
+
+    /**
+     * Checks if the signature of a self generated license with older version needs to be
+     * recreated with the new key
+     */
+    public static boolean signatureNeedsUpdate(License license) {
+        return ("basic".equals(license.type()) || "trial".equals(license.type())) &&
+                (license.version() < License.VERSION_CRYPTO_ALGORITHMS);
+    }
 }