diff --git a/src/lib/prov/pkcs11/p11_ecc_key.cpp b/src/lib/prov/pkcs11/p11_ecc_key.cpp
index da8d12a1863..3101344d4ad 100644
--- a/src/lib/prov/pkcs11/p11_ecc_key.cpp
+++ b/src/lib/prov/pkcs11/p11_ecc_key.cpp
@@ -13,16 +13,24 @@
 #if defined(BOTAN_HAS_ECC_PUBLIC_KEY_CRYPTO)
 
    #include <botan/ber_dec.h>
+   #include <botan/internal/ec_key_data.h>
    #include <botan/internal/workfactor.h>
 
 namespace Botan::PKCS11 {
+
 namespace {
+
 /// Converts a DER-encoded ANSI X9.62 ECPoint to EC_Point
-EC_Point decode_public_point(const secure_vector<uint8_t>& ec_point_data, const EC_Group& group) {
-   secure_vector<uint8_t> ec_point;
+EC_AffinePoint decode_public_point(const EC_Group& group, std::span<const uint8_t> ec_point_data) {
+   std::vector<uint8_t> ec_point;
    BER_Decoder(ec_point_data).decode(ec_point, ASN1_Type::OctetString);
-   return group.OS2ECP(ec_point);
+   if(auto pt = EC_AffinePoint::deserialize(group, ec_point)) {
+      return *pt;
+   } else {
+      throw Decoding_Error("Encoded EC public point was invalid");
+   }
 }
+
 }  // namespace
 
 EC_PublicKeyGenerationProperties::EC_PublicKeyGenerationProperties(const std::vector<uint8_t>& ec_params) :
@@ -38,20 +46,19 @@ EC_PublicKeyImportProperties::EC_PublicKeyImportProperties(const std::vector<uin
 }
 
 PKCS11_EC_PublicKey::PKCS11_EC_PublicKey(Session& session, ObjectHandle handle) : Object(session, handle) {
-   secure_vector<uint8_t> ec_parameters = get_attribute_value(AttributeType::EcParams);
-   m_domain_params = EC_Group(unlock(ec_parameters));
-   m_public_key = decode_public_point(get_attribute_value(AttributeType::EcPoint), m_domain_params);
-   m_domain_encoding = EC_Group_Encoding::NamedCurve;
+   auto ec_parameters = get_attribute_value(AttributeType::EcParams);
+   auto pt_bytes = get_attribute_value(AttributeType::EcPoint);
+
+   EC_Group group(ec_parameters);
+   auto pt = decode_public_point(group, pt_bytes);
+   m_public_key = std::make_shared<EC_PublicKey_Data>(group, pt);
 }
 
 PKCS11_EC_PublicKey::PKCS11_EC_PublicKey(Session& session, const EC_PublicKeyImportProperties& props) :
       Object(session, props) {
-   m_domain_params = EC_Group(props.ec_params());
-
-   secure_vector<uint8_t> ec_point;
-   BER_Decoder(props.ec_point()).decode(ec_point, ASN1_Type::OctetString);
-   m_public_key = m_domain_params.OS2ECP(ec_point);
-   m_domain_encoding = EC_Group_Encoding::NamedCurve;
+   EC_Group group(props.ec_params());
+   auto pt = decode_public_point(group, props.ec_point());
+   m_public_key = std::make_shared<EC_PublicKey_Data>(group, pt);
 }
 
 EC_PrivateKeyImportProperties::EC_PrivateKeyImportProperties(const std::vector<uint8_t>& ec_params,
@@ -61,8 +68,7 @@ EC_PrivateKeyImportProperties::EC_PrivateKeyImportProperties(const std::vector<u
    add_binary(AttributeType::Value, m_value.serialize());
 }
 
-PKCS11_EC_PrivateKey::PKCS11_EC_PrivateKey(Session& session, ObjectHandle handle) :
-      Object(session, handle), m_domain_params(), m_public_key() {
+PKCS11_EC_PrivateKey::PKCS11_EC_PrivateKey(Session& session, ObjectHandle handle) : Object(session, handle) {
    secure_vector<uint8_t> ec_parameters = get_attribute_value(AttributeType::EcParams);
    m_domain_params = EC_Group(unlock(ec_parameters));
 }
@@ -96,9 +102,10 @@ PKCS11_EC_PrivateKey::PKCS11_EC_PrivateKey(Session& session,
                                        &priv_key_handle);
 
    this->reset_handle(priv_key_handle);
-
    Object public_key(session, pub_key_handle);
-   m_public_key = decode_public_point(public_key.get_attribute_value(AttributeType::EcPoint), m_domain_params);
+
+   auto pt_bytes = public_key.get_attribute_value(AttributeType::EcPoint);
+   m_public_key = decode_public_point(m_domain_params, pt_bytes).to_legacy_point();
 }
 
 size_t PKCS11_EC_PrivateKey::key_length() const {
diff --git a/src/lib/pubkey/ec_group/ec_apoint.cpp b/src/lib/pubkey/ec_group/ec_apoint.cpp
index 1baa53bfc36..450e716ae34 100644
--- a/src/lib/pubkey/ec_group/ec_apoint.cpp
+++ b/src/lib/pubkey/ec_group/ec_apoint.cpp
@@ -65,8 +65,11 @@ EC_AffinePoint EC_AffinePoint::hash_to_curve_nu(const EC_Group& group,
 EC_AffinePoint::~EC_AffinePoint() = default;
 
 std::optional<EC_AffinePoint> EC_AffinePoint::deserialize(const EC_Group& group, std::span<const uint8_t> bytes) {
-   auto pt = group._data()->point_deserialize(bytes);
-   return EC_AffinePoint(std::move(pt));
+   if(auto pt = group._data()->point_deserialize(bytes)) {
+      return EC_AffinePoint(std::move(pt));
+   } else {
+      return {};
+   }
 }
 
 EC_AffinePoint EC_AffinePoint::g_mul(const EC_Scalar& scalar, RandomNumberGenerator& rng, std::vector<BigInt>& ws) {
diff --git a/src/lib/pubkey/ec_group/ec_group.cpp b/src/lib/pubkey/ec_group/ec_group.cpp
index a6ace709735..e7a7cbc265e 100644
--- a/src/lib/pubkey/ec_group/ec_group.cpp
+++ b/src/lib/pubkey/ec_group/ec_group.cpp
@@ -210,9 +210,15 @@ std::pair<std::shared_ptr<EC_Group_Data>, bool> EC_Group::BER_decode_EC_group(st
    BER_Object obj = ber.get_next_object();
 
    if(obj.type() == ASN1_Type::ObjectId) {
-      OID dom_par_oid;
-      BER_Decoder(bits).decode(dom_par_oid);
-      return std::make_pair(ec_group_data().lookup(dom_par_oid), false);
+      OID oid;
+      BER_Decoder(bits).decode(oid);
+
+      auto data = ec_group_data().lookup(oid);
+      if(!data) {
+         throw Decoding_Error(fmt("Unknown namedCurve OID '{}'", oid.to_string()));
+      }
+
+      return std::make_pair(data, false);
    }
 
    if(obj.type() == ASN1_Type::Sequence) {
diff --git a/src/lib/pubkey/ec_group/ec_scalar.cpp b/src/lib/pubkey/ec_group/ec_scalar.cpp
index 7021b76b601..81ff0e73cca 100644
--- a/src/lib/pubkey/ec_group/ec_scalar.cpp
+++ b/src/lib/pubkey/ec_group/ec_scalar.cpp
@@ -63,6 +63,12 @@ EC_Scalar EC_Scalar::from_bigint(const EC_Group& group, const BigInt& bn) {
    return EC_Scalar(group._data()->scalar_from_bigint(bn));
 }
 
+BigInt EC_Scalar::to_bigint() const {
+   secure_vector<uint8_t> bytes(m_scalar->bytes());
+   m_scalar->serialize_to(bytes);
+   return BigInt::from_bytes(bytes);
+}
+
 EC_Scalar EC_Scalar::gk_x_mod_order(const EC_Scalar& scalar, RandomNumberGenerator& rng, std::vector<BigInt>& ws) {
    const auto& group = scalar._inner().group();
    return EC_Scalar(group->gk_x_mod_order(scalar.inner(), rng, ws));
@@ -106,6 +112,13 @@ std::optional<EC_Scalar> EC_Scalar::deserialize(const EC_Group& group, std::span
    }
 }
 
+EC_Scalar::EC_Scalar(const EC_Group& group, std::span<const uint8_t> bytes) {
+   m_scalar = group._data()->scalar_deserialize(bytes);
+   if(!m_scalar) {
+      throw Decoding_Error("Failed to deserialize elliptic curve scalar");
+   }
+}
+
 bool EC_Scalar::is_zero() const {
    return inner().is_zero();
 }
diff --git a/src/lib/pubkey/ec_group/ec_scalar.h b/src/lib/pubkey/ec_group/ec_scalar.h
index fc8e21b426d..359cb972693 100644
--- a/src/lib/pubkey/ec_group/ec_scalar.h
+++ b/src/lib/pubkey/ec_group/ec_scalar.h
@@ -68,6 +68,14 @@ class BOTAN_UNSTABLE_API EC_Scalar final {
       */
       static EC_Scalar random(const EC_Group& group, RandomNumberGenerator& rng);
 
+      /**
+      * Convert a bytestring to an EC_Scalar
+      *
+      * This is similar to deserialize but instead of returning nullopt if the input
+      * is invalid, it will throw an exception.
+      */
+      EC_Scalar(const EC_Group& group, std::span<const uint8_t> bytes);
+
       /**
       * Return the scalar value 1
       */
@@ -180,6 +188,16 @@ class BOTAN_UNSTABLE_API EC_Scalar final {
       */
       bool is_eq(const EC_Scalar& x) const;
 
+      /**
+      * Convert *this to a BigInt
+      */
+      BigInt to_bigint() const;
+
+      /**
+      * Return the underlying group
+      */
+      const EC_Group& group() const;
+
       friend EC_Scalar operator+(const EC_Scalar& x, const EC_Scalar& y) { return x.add(y); }
 
       friend EC_Scalar operator-(const EC_Scalar& x, const EC_Scalar& y) { return x.sub(y); }
diff --git a/src/lib/pubkey/ecc_key/ec_key_data.cpp b/src/lib/pubkey/ecc_key/ec_key_data.cpp
new file mode 100644
index 00000000000..38f618ef35c
--- /dev/null
+++ b/src/lib/pubkey/ecc_key/ec_key_data.cpp
@@ -0,0 +1,58 @@
+/*
+* (C) 2024 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/internal/ec_key_data.h>
+
+namespace Botan {
+
+EC_PublicKey_Data::EC_PublicKey_Data(const EC_Group& group, std::span<const uint8_t> bytes) :
+      m_group(group),
+      m_point(EC_AffinePoint(group, bytes)),
+      m_legacy_point(m_point.to_legacy_point()) {}
+
+EC_PrivateKey_Data::EC_PrivateKey_Data(const EC_Group& group, RandomNumberGenerator& rng) :
+      m_group(group), m_scalar(EC_Scalar::random(m_group, rng)), m_legacy_x(m_scalar.to_bigint()) {}
+
+EC_PrivateKey_Data::EC_PrivateKey_Data(const EC_Group& group, const BigInt& x) :
+      m_group(group), m_scalar(EC_Scalar::from_bigint(m_group, x)), m_legacy_x(m_scalar.to_bigint()) {}
+
+EC_PrivateKey_Data::EC_PrivateKey_Data(const EC_Group& group, std::span<const uint8_t> bytes) :
+      m_group(group), m_scalar(m_group, bytes), m_legacy_x(m_scalar.to_bigint()) {}
+
+std::shared_ptr<EC_PublicKey_Data> EC_PrivateKey_Data::public_key(RandomNumberGenerator& rng,
+                                                                  bool with_modular_inverse) const {
+   auto public_point = [&]() {
+      std::vector<BigInt> ws;
+      if(with_modular_inverse) {
+         return EC_AffinePoint::g_mul(m_scalar.invert(), rng, ws);
+      } else {
+         return EC_AffinePoint::g_mul(m_scalar, rng, ws);
+      }
+   };
+
+   return std::make_shared<EC_PublicKey_Data>(m_group, public_point());
+}
+
+std::shared_ptr<EC_PublicKey_Data> EC_PrivateKey_Data::public_key(bool with_modular_inverse) const {
+   // TODO this is using unblinded Montgomery ladder since we don't have an RNG
+   // Probably need to handle Null_RNG in g_mul
+   auto public_point = [&]() {
+      std::vector<BigInt> ws;
+      if(with_modular_inverse) {
+         return EC_AffinePoint(group(), group().get_base_point() * m_scalar.invert().to_bigint());
+      } else {
+         return EC_AffinePoint(group(), group().get_base_point() * m_scalar.to_bigint());
+      }
+   };
+
+   return std::make_shared<EC_PublicKey_Data>(m_group, public_point());
+}
+
+void EC_PrivateKey_Data::serialize_to(std::span<uint8_t> output) const {
+   m_scalar.serialize_to(output);
+}
+
+}  // namespace Botan
diff --git a/src/lib/pubkey/ecc_key/ec_key_data.h b/src/lib/pubkey/ecc_key/ec_key_data.h
new file mode 100644
index 00000000000..2feef249fb4
--- /dev/null
+++ b/src/lib/pubkey/ecc_key/ec_key_data.h
@@ -0,0 +1,76 @@
+/*
+* (C) 2024 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_EC_KEY_DATA_H_
+#define BOTAN_EC_KEY_DATA_H_
+
+#include <botan/ec_apoint.h>
+#include <botan/ec_group.h>
+#include <botan/ec_scalar.h>
+
+#include <botan/bigint.h>
+#include <botan/ec_point.h>
+
+namespace Botan {
+
+class RandomNumberGenerator;
+
+class EC_PublicKey_Data final {
+   public:
+      EC_PublicKey_Data(EC_Group group, EC_AffinePoint pt) :
+            m_group(std::move(group)), m_point(std::move(pt)), m_legacy_point(m_point.to_legacy_point()) {}
+
+      EC_PublicKey_Data(const EC_Group& group, std::span<const uint8_t> bytes);
+
+      const EC_Group& group() const { return m_group; }
+
+      const EC_AffinePoint& public_key() const { return m_point; }
+
+      const EC_Point& legacy_point() const { return m_legacy_point; }
+
+   private:
+      EC_Group m_group;
+      EC_AffinePoint m_point;
+      EC_Point m_legacy_point;
+};
+
+class EC_PrivateKey_Data final {
+   public:
+      EC_PrivateKey_Data(const EC_Group& group, RandomNumberGenerator& rng);
+
+      EC_PrivateKey_Data(const EC_Group& group, const BigInt& x);
+
+      EC_PrivateKey_Data(const EC_Group& group, std::span<const uint8_t> bytes);
+
+      std::shared_ptr<EC_PublicKey_Data> public_key(RandomNumberGenerator& rng, bool with_modular_inverse) const;
+
+      std::shared_ptr<EC_PublicKey_Data> public_key(bool with_modular_inverse) const;
+
+      void serialize_to(std::span<uint8_t> output) const;
+
+      template <typename T>
+      T serialize() const {
+         T bytes(this->group().get_order_bytes());
+         this->serialize_to(bytes);
+         return bytes;
+      }
+
+      const EC_Group& group() const { return m_group; }
+
+      const EC_Scalar& private_key() const { return m_scalar; }
+
+      const BigInt& legacy_bigint() const { return m_legacy_x; }
+
+   private:
+      EC_Group m_group;
+
+      EC_Scalar m_scalar;
+      BigInt m_legacy_x;
+};
+
+}  // namespace Botan
+
+#endif
diff --git a/src/lib/pubkey/ecc_key/ecc_key.cpp b/src/lib/pubkey/ecc_key/ecc_key.cpp
index 0a11728520b..1213325e0c8 100644
--- a/src/lib/pubkey/ecc_key/ecc_key.cpp
+++ b/src/lib/pubkey/ecc_key/ecc_key.cpp
@@ -14,6 +14,7 @@
 #include <botan/ec_point.h>
 #include <botan/numthry.h>
 #include <botan/secmem.h>
+#include <botan/internal/ec_key_data.h>
 #include <botan/internal/fmt.h>
 #include <botan/internal/workfactor.h>
 
@@ -29,7 +30,7 @@ size_t EC_PublicKey::estimated_strength() const {
 
 namespace {
 
-EC_Group_Encoding default_encoding_for(EC_Group& group) {
+EC_Group_Encoding default_encoding_for(const EC_Group& group) {
    if(group.get_curve_oid().empty()) {
       return EC_Group_Encoding::Explicit;
    } else {
@@ -39,16 +40,34 @@ EC_Group_Encoding default_encoding_for(EC_Group& group) {
 
 }  // namespace
 
-EC_PublicKey::EC_PublicKey(const EC_Group& dom_par, const EC_Point& pub_point) :
-      m_domain_params(dom_par), m_public_key(pub_point), m_domain_encoding(default_encoding_for(m_domain_params)) {}
+EC_PublicKey::EC_PublicKey(const EC_Group& group, const EC_Point& pub_point) {
+   auto pt = EC_AffinePoint(group, pub_point);
+   m_public_key = std::make_shared<EC_PublicKey_Data>(group, pt);
+   m_domain_encoding = default_encoding_for(domain());
+}
 
-EC_PublicKey::EC_PublicKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits) :
-      m_domain_params{EC_Group(alg_id.parameters())},
-      m_public_key{domain().OS2ECP(key_bits)},
-      m_domain_encoding(default_encoding_for(m_domain_params)) {}
+EC_PublicKey::EC_PublicKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits) {
+   m_public_key = std::make_shared<EC_PublicKey_Data>(EC_Group(alg_id.parameters()), key_bits);
+   m_domain_encoding = default_encoding_for(domain());
+}
+
+const EC_Group& EC_PublicKey::domain() const {
+   BOTAN_STATE_CHECK(m_public_key != nullptr);
+   return m_public_key->group();
+}
+
+const EC_Point& EC_PublicKey::public_point() const {
+   BOTAN_STATE_CHECK(m_public_key != nullptr);
+   return m_public_key->legacy_point();
+}
+
+const EC_AffinePoint& EC_PublicKey::_public_key() const {
+   BOTAN_STATE_CHECK(m_public_key != nullptr);
+   return m_public_key->public_key();
+}
 
 bool EC_PublicKey::check_key(RandomNumberGenerator& rng, bool /*strong*/) const {
-   return m_domain_params.verify_group(rng) && m_domain_params.verify_public_element(public_point());
+   return domain().verify_group(rng) && domain().verify_public_element(public_point());
 }
 
 AlgorithmIdentifier EC_PublicKey::algorithm_identifier() const {
@@ -76,7 +95,7 @@ void EC_PublicKey::set_point_encoding(EC_Point_Format enc) {
 }
 
 void EC_PublicKey::set_parameter_encoding(EC_Group_Encoding form) {
-   if(form == EC_Group_Encoding::NamedCurve && m_domain_params.get_curve_oid().empty()) {
+   if(form == EC_Group_Encoding::NamedCurve && domain().get_curve_oid().empty()) {
       throw Invalid_Argument("Cannot used NamedCurve encoding for a curve without an OID");
    }
 
@@ -84,11 +103,16 @@ void EC_PublicKey::set_parameter_encoding(EC_Group_Encoding form) {
 }
 
 const BigInt& EC_PrivateKey::private_value() const {
-   if(m_private_key == 0) {
+   if(!m_private_key) {
       throw Invalid_State("EC_PrivateKey::private_value - uninitialized");
    }
 
-   return m_private_key;
+   return m_private_key->legacy_bigint();
+}
+
+const EC_Scalar& EC_PrivateKey::_private_key() const {
+   BOTAN_STATE_CHECK(m_private_key != nullptr);
+   return m_private_key->private_key();
 }
 
 /**
@@ -98,39 +122,30 @@ EC_PrivateKey::EC_PrivateKey(RandomNumberGenerator& rng,
                              const EC_Group& ec_group,
                              const BigInt& x,
                              bool with_modular_inverse) {
-   m_domain_params = ec_group;
-   m_domain_encoding = default_encoding_for(m_domain_params);
-
    if(x == 0) {
-      m_private_key = ec_group.random_scalar(rng);
+      m_private_key = std::make_shared<EC_PrivateKey_Data>(ec_group, rng);
    } else {
-      BOTAN_ARG_CHECK(x > 0 && x < ec_group.get_order(), "ECC private key out of range");
-      m_private_key = x;
+      m_private_key = std::make_shared<EC_PrivateKey_Data>(ec_group, x);
    }
 
-   std::vector<BigInt> ws;
-
-   if(with_modular_inverse) {
-      // ECKCDSA
-      m_public_key = domain().blinded_base_point_multiply(m_domain_params.inverse_mod_order(m_private_key), rng, ws);
-   } else {
-      m_public_key = domain().blinded_base_point_multiply(m_private_key, rng, ws);
-   }
-
-   BOTAN_ASSERT(m_public_key.on_the_curve(), "Generated public key point was on the curve");
+   m_public_key = m_private_key->public_key(rng, with_modular_inverse);
+   m_domain_encoding = default_encoding_for(domain());
 }
 
 secure_vector<uint8_t> EC_PrivateKey::raw_private_key_bits() const {
-   return m_private_key.serialize<secure_vector<uint8_t>>(domain().get_order_bytes());
+   BOTAN_STATE_CHECK(m_private_key != nullptr);
+   return m_private_key->serialize<secure_vector<uint8_t>>();
 }
 
 secure_vector<uint8_t> EC_PrivateKey::private_key_bits() const {
+   BOTAN_STATE_CHECK(m_private_key != nullptr && m_public_key != nullptr);
+
    return DER_Encoder()
       .start_sequence()
       .encode(static_cast<size_t>(1))
       .encode(raw_private_key_bits(), ASN1_Type::OctetString)
       .start_explicit_context_specific(1)
-      .encode(m_public_key.encode(EC_Point_Format::Uncompressed), ASN1_Type::BitString)
+      .encode(m_public_key->public_key().serialize_uncompressed(), ASN1_Type::BitString)
       .end_cons()
       .end_cons()
       .get_contents();
@@ -139,41 +154,33 @@ secure_vector<uint8_t> EC_PrivateKey::private_key_bits() const {
 EC_PrivateKey::EC_PrivateKey(const AlgorithmIdentifier& alg_id,
                              std::span<const uint8_t> key_bits,
                              bool with_modular_inverse) {
-   m_domain_params = EC_Group(alg_id.parameters());
-   m_domain_encoding = default_encoding_for(m_domain_params);
+   EC_Group group(alg_id.parameters());
 
    OID key_parameters;
+   secure_vector<uint8_t> private_key_bits;
    secure_vector<uint8_t> public_key_bits;
 
    BER_Decoder(key_bits)
       .start_sequence()
       .decode_and_check<size_t>(1, "Unknown version code for ECC key")
-      .decode_octet_string_bigint(m_private_key)
+      .decode(private_key_bits, ASN1_Type::OctetString)
       .decode_optional(key_parameters, ASN1_Type(0), ASN1_Class::ExplicitContextSpecific)
       .decode_optional_string(public_key_bits, ASN1_Type::BitString, 1, ASN1_Class::ExplicitContextSpecific)
       .end_cons();
 
-   if(m_private_key < 1 || m_private_key >= m_domain_params.get_order()) {
-      throw Decoding_Error("Invalid EC private key");
-   }
+   m_private_key = std::make_shared<EC_PrivateKey_Data>(group, private_key_bits);
 
    if(public_key_bits.empty()) {
-      if(with_modular_inverse) {
-         // ECKCDSA
-         m_public_key = domain().get_base_point() * m_domain_params.inverse_mod_order(m_private_key);
-      } else {
-         m_public_key = domain().get_base_point() * m_private_key;
-      }
-
-      BOTAN_ASSERT(m_public_key.on_the_curve(), "Public point derived from loaded key was on the curve");
+      m_public_key = m_private_key->public_key(with_modular_inverse);
    } else {
-      m_public_key = domain().OS2ECP(public_key_bits);
-      // OS2ECP verifies that the point is on the curve
+      m_public_key = std::make_shared<EC_PublicKey_Data>(group, public_key_bits);
    }
+
+   m_domain_encoding = default_encoding_for(domain());
 }
 
 bool EC_PrivateKey::check_key(RandomNumberGenerator& rng, bool strong) const {
-   if(m_private_key < 1 || m_private_key >= m_domain_params.get_order()) {
+   if(!m_private_key) {
       return false;
    }
 
diff --git a/src/lib/pubkey/ecc_key/ecc_key.h b/src/lib/pubkey/ecc_key/ecc_key.h
index d2eb4da54ae..36194b550f0 100644
--- a/src/lib/pubkey/ecc_key/ecc_key.h
+++ b/src/lib/pubkey/ecc_key/ecc_key.h
@@ -12,9 +12,13 @@
 
 #include <botan/ec_group.h>
 #include <botan/pk_keys.h>
+#include <memory>
 
 namespace Botan {
 
+class EC_PublicKey_Data;
+class EC_PrivateKey_Data;
+
 /**
 * This class represents abstract ECC public keys. When encoding a key
 * via an encoder that can be accessed via the corresponding member
@@ -37,7 +41,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
       * domain parameters of this point are not set
       * @result the public point of this key
       */
-      const EC_Point& public_point() const { return m_public_key; }
+      BOTAN_DEPRECATED("Deprecated use raw_public_key_bits") const EC_Point& public_point() const;
 
       AlgorithmIdentifier algorithm_identifier() const override;
 
@@ -53,7 +57,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
       * domain parameters of this point are not set
       * @result the domain parameters of this key
       */
-      const EC_Group& domain() const { return m_domain_params; }
+      const EC_Group& domain() const;
 
       /**
       * Set the domain parameter encoding to be used when encoding this key.
@@ -94,13 +98,14 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
 
       const BigInt& get_int_field(std::string_view field) const override;
 
+      const EC_AffinePoint& _public_key() const;
    protected:
       /**
       * Create a public key.
-      * @param dom_par EC domain parameters
+      * @param group EC domain parameters
       * @param pub_point public point on the curve
       */
-      EC_PublicKey(const EC_Group& dom_par, const EC_Point& pub_point);
+      EC_PublicKey(const EC_Group& group, const EC_Point& pub_point);
 
       /**
       * Load a public key.
@@ -109,11 +114,10 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
       */
       EC_PublicKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits);
 
-      EC_PublicKey() : m_domain_params{}, m_public_key{}, m_domain_encoding(EC_Group_Encoding::Explicit) {}
+      EC_PublicKey() = default;
 
-      EC_Group m_domain_params;
-      EC_Point m_public_key;
-      EC_Group_Encoding m_domain_encoding;
+      std::shared_ptr<EC_PublicKey_Data> m_public_key;
+      EC_Group_Encoding m_domain_encoding = EC_Group_Encoding::NamedCurve;
       EC_Point_Format m_point_encoding = EC_Point_Format::Uncompressed;
 };
 
@@ -137,7 +141,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PrivateKey : public virtual EC_PublicKey,
       * Get the private key value of this key object.
       * @result the private key value of this key object
       */
-      const BigInt& private_value() const;
+      BOTAN_DEPRECATED("Deprecated use get_int_field or raw_private_key_bits") const BigInt& private_value() const;
 
       EC_PrivateKey(const EC_PrivateKey& other) = default;
       EC_PrivateKey& operator=(const EC_PrivateKey& other) = default;
@@ -145,6 +149,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PrivateKey : public virtual EC_PublicKey,
 
       const BigInt& get_int_field(std::string_view field) const final;
 
+      const EC_Scalar& _private_key() const;
    protected:
       /*
       * If x=0, creates a new private key in the domain
@@ -174,7 +179,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PrivateKey : public virtual EC_PublicKey,
 
       EC_PrivateKey() = default;
 
-      BigInt m_private_key;
+      std::shared_ptr<const EC_PrivateKey_Data> m_private_key;
 };
 
 BOTAN_DIAGNOSTIC_POP
diff --git a/src/lib/pubkey/ecc_key/info.txt b/src/lib/pubkey/ecc_key/info.txt
index 329bb06c3ff..982413ea900 100644
--- a/src/lib/pubkey/ecc_key/info.txt
+++ b/src/lib/pubkey/ecc_key/info.txt
@@ -18,3 +18,7 @@ numbertheory
 <header:public>
 ecc_key.h
 </header:public>
+
+<header:internal>
+ec_key_data.h
+</header:internal>
diff --git a/src/lib/pubkey/gost_3410/gost_3410.cpp b/src/lib/pubkey/gost_3410/gost_3410.cpp
index f8740fc4cce..ce8a9f610ea 100644
--- a/src/lib/pubkey/gost_3410/gost_3410.cpp
+++ b/src/lib/pubkey/gost_3410/gost_3410.cpp
@@ -11,6 +11,7 @@
 
 #include <botan/ber_dec.h>
 #include <botan/der_enc.h>
+#include <botan/internal/ec_key_data.h>
 #include <botan/internal/fmt.h>
 #include <botan/internal/pk_ops_impl.h>
 
@@ -59,14 +60,14 @@ GOST_3410_PublicKey::GOST_3410_PublicKey(const AlgorithmIdentifier& alg_id, std:
    // The parameters also includes hash and cipher OIDs
    BER_Decoder(alg_id.parameters()).start_sequence().decode(ecc_param_id);
 
-   m_domain_params = EC_Group::from_OID(ecc_param_id);
+   auto group = EC_Group::from_OID(ecc_param_id);
 
-   const size_t p_bits = m_domain_params.get_p_bits();
+   const size_t p_bits = group.get_p_bits();
    if(p_bits != 256 && p_bits != 512) {
       throw Decoding_Error(fmt("GOST-34.10-2012 is not defined for parameters of size {}", p_bits));
    }
 
-   secure_vector<uint8_t> bits;
+   std::vector<uint8_t> bits;
    BER_Decoder(key_bits).decode(bits, ASN1_Type::OctetString);
 
    if(bits.size() != 2 * (p_bits / 8)) {
@@ -76,22 +77,18 @@ GOST_3410_PublicKey::GOST_3410_PublicKey(const AlgorithmIdentifier& alg_id, std:
    const size_t part_size = bits.size() / 2;
 
    // Keys are stored in little endian format (WTF)
-   for(size_t i = 0; i != part_size / 2; ++i) {
-      std::swap(bits[i], bits[part_size - 1 - i]);
-      std::swap(bits[part_size + i], bits[2 * part_size - 1 - i]);
-   }
-
-   BigInt x(bits.data(), part_size);
-   BigInt y(&bits[part_size], part_size);
-
-   m_public_key = domain().point(x, y);
+   std::vector<uint8_t> encoding;
+   encoding.reserve(bits.size() + 1);
+   encoding.push_back(0x04);
+   encoding.insert(encoding.end(), bits.rbegin() + part_size, bits.rend());
+   encoding.insert(encoding.end(), bits.rbegin(), bits.rend() - part_size);
 
-   BOTAN_ASSERT(m_public_key.on_the_curve(), "Loaded GOST 34.10 public key is on the curve");
+   m_public_key = std::make_shared<EC_PublicKey_Data>(group, encoding);
 }
 
 GOST_3410_PrivateKey::GOST_3410_PrivateKey(RandomNumberGenerator& rng, const EC_Group& domain, const BigInt& x) :
       EC_PrivateKey(rng, domain, x) {
-   const size_t p_bits = m_domain_params.get_p_bits();
+   const size_t p_bits = domain.get_p_bits();
    if(p_bits != 256 && p_bits != 512) {
       throw Decoding_Error(fmt("GOST-34.10-2012 is not defined for parameters of size {}", p_bits));
    }
diff --git a/src/lib/pubkey/sm2/sm2.cpp b/src/lib/pubkey/sm2/sm2.cpp
index d3f04ca8063..bfedd06ef32 100644
--- a/src/lib/pubkey/sm2/sm2.cpp
+++ b/src/lib/pubkey/sm2/sm2.cpp
@@ -32,8 +32,10 @@ bool SM2_PrivateKey::check_key(RandomNumberGenerator& rng, bool strong) const {
 
    // SM2 has an oddity in private key generation when compared to
    // other EC*DSA style signature algorithms described in ISO14888-3:
-   // the private key x MUST be in ]0, q-1[ instead of ]0, q[.
-   if(m_private_key < 1 || m_private_key >= m_domain_params.get_order() - 1) {
+   // the private key x MUST be in [0, q-1) instead of [0, q).
+   //
+   // The lower bound is already checked by the default impl
+   if(private_value() >= domain().get_order() - 1) {
       return false;
    }
 
@@ -46,12 +48,12 @@ bool SM2_PrivateKey::check_key(RandomNumberGenerator& rng, bool strong) const {
 
 SM2_PrivateKey::SM2_PrivateKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits) :
       EC_PrivateKey(alg_id, key_bits) {
-   m_da_inv = domain().inverse_mod_order(m_private_key + 1);
+   m_da_inv = domain().inverse_mod_order(private_value() + 1);
 }
 
 SM2_PrivateKey::SM2_PrivateKey(RandomNumberGenerator& rng, const EC_Group& domain, const BigInt& x) :
       EC_PrivateKey(rng, domain, x) {
-   m_da_inv = domain.inverse_mod_order(m_private_key + 1);
+   m_da_inv = domain.inverse_mod_order(private_value() + 1);
 }
 
 std::vector<uint8_t> sm2_compute_za(HashFunction& hash,
diff --git a/src/tests/test_ecdsa.cpp b/src/tests/test_ecdsa.cpp
index d20d90d3bbc..5aaee63c2f3 100644
--- a/src/tests/test_ecdsa.cpp
+++ b/src/tests/test_ecdsa.cpp
@@ -234,8 +234,12 @@ class ECDSA_Invalid_Key_Tests final : public Text_Based_Test {
             return result;
          }
 
-         auto key = std::make_unique<Botan::ECDSA_PublicKey>(group, *public_point);
-         result.test_eq("public key fails check", key->check_key(this->rng(), false), false);
+         try {
+            auto key = std::make_unique<Botan::ECDSA_PublicKey>(group, *public_point);
+            result.test_failure("Invalid public key was deserialized");
+         } catch(Botan::Decoding_Error&) {
+            result.test_success("public key fails to deserialize");
+         }
          return result;
       }
 };