diff --git a/src/pydata_sphinx_theme/__init__.py b/src/pydata_sphinx_theme/__init__.py
index 5215b4758..643563590 100644
--- a/src/pydata_sphinx_theme/__init__.py
+++ b/src/pydata_sphinx_theme/__init__.py
@@ -54,6 +54,35 @@ def update_config(app, env):
             )
         )
 
+    # Add an analytics ID to the site if provided
+    # Currently only supports the two types of Google Analytics id.
+    gid = theme_options.get("google_analytics_id")
+    if gid:
+        # In this case it is "new-style" google analytics
+        if "G-" in gid:
+            gid_js_path = f"https://www.googletagmanager.com/gtag/js?id={gid}"
+            gid_script = f"""
+                window.dataLayer = window.dataLayer || [];
+                function gtag(){{ dataLayer.push(arguments); }}
+                gtag('js', new Date());
+                gtag('config', '{gid}');
+            """
+        # In this case it is "old-style" google analytics
+        else:
+            gid_js_path = "https://www.google-analytics.com/analytics.js"
+            gid_script = f"""
+                window.ga = window.ga || function () {{
+                    (ga.q = ga.q || []).push(arguments) }};
+                ga.l = +new Date;
+                ga('create', '{gid}', 'auto');
+                ga('set', 'anonymizeIp', true);
+                ga('send', 'pageview');
+            """
+
+        # Link the JS files
+        app.add_js_file(gid_js_path, loading_method="async")
+        app.add_js_file(None, body=gid_script)
+
 
 def update_templates(app, pagename, templatename, context, doctree):
     """Update template names and assets for page build."""
@@ -88,6 +117,17 @@ def update_templates(app, pagename, templatename, context, doctree):
         if theme_css_name in context["css_files"]:
             context["css_files"].remove(theme_css_name)
 
+    # Add links for favicons in the topbar
+    for favicon in context.get("theme_favicons", []):
+        icon_type = Path(favicon["href"]).suffix.strip(".")
+        # Sphinx will auto-resolve href if it's a local file
+        app.add_css_file(
+            favicon["href"],
+            rel=favicon["rel"],
+            sizes=favicon["sizes"],
+            **{"type": f"image/{icon_type}"},
+        )
+
 
 def add_toctree_functions(app, pagename, templatename, context, doctree):
     """Add functions so Jinja templates can add toctree objects."""
@@ -259,46 +299,9 @@ def navbar_align_class():
             )
         return align_options[align]
 
-    def generate_google_analytics_script(id):
-        """Handle the two types of google analytics id."""
-        if id:
-            if "G-" in id:
-                script = f"""
-                <script
-                    async
-                    src='https://www.googletagmanager.com/gtag/js?id={id}'
-                ></script>
-                <script>
-                    window.dataLayer = window.dataLayer || [];
-                    function gtag(){{ dataLayer.push(arguments); }}
-                    gtag('js', new Date());
-                    gtag('config', '{id}');
-                </script>
-                """
-            else:
-                script = f"""
-                    <script
-                        async
-                        src='https://www.google-analytics.com/analytics.js'
-                    ></script>
-                    <script>
-                        window.ga = window.ga || function () {{
-                            (ga.q = ga.q || []).push(arguments) }};
-                        ga.l = +new Date;
-                        ga('create', '{id}', 'auto');
-                        ga('set', 'anonymizeIp', true);
-                        ga('send', 'pageview');
-                    </script>
-                """
-            soup = bs(script, "html.parser")
-            return soup
-        else:
-            return ""
-
     context["generate_nav_html"] = generate_nav_html
     context["generate_toc_html"] = generate_toc_html
     context["navbar_align_class"] = navbar_align_class
-    context["generate_google_analytics_script"] = generate_google_analytics_script
 
 
 def _add_collapse_checkboxes(soup):
diff --git a/src/pydata_sphinx_theme/theme/pydata_sphinx_theme/layout.html b/src/pydata_sphinx_theme/theme/pydata_sphinx_theme/layout.html
index 5ace4bc88..b4d9c69c2 100644
--- a/src/pydata_sphinx_theme/theme/pydata_sphinx_theme/layout.html
+++ b/src/pydata_sphinx_theme/theme/pydata_sphinx_theme/layout.html
@@ -17,16 +17,6 @@
 {%- block extrahead %}
 <meta name="viewport" content="width=device-width, initial-scale=1" />
 <meta name="docsearch:language" content="{{ language }}">
-{% for favicon in theme_favicons %}
-  {% if favicon.href[:4] == 'http'%}
-  <link rel="{{ favicon.rel }}" sizes="{{ favicon.sizes }}" href="{{ favicon.href }}">
-  {% else %}
-  <link rel="{{ favicon.rel }}" sizes="{{ favicon.sizes }}" href="{{ pathto('_static/' + favicon.href, 1) }}">
-  {% endif %}
-{% endfor %}
-
-<!-- Google Analytics -->
-{{ generate_google_analytics_script(id=theme_google_analytics_id) }}
 {%- endblock %}
 
 {% block body_tag %}
diff --git a/tests/test_build.py b/tests/test_build.py
index 3067d5dfe..c43592018 100644
--- a/tests/test_build.py
+++ b/tests/test_build.py
@@ -185,14 +185,16 @@ def test_favicons(sphinx_build_factory):
 
     icon_16 = (
         '<link href="https://secure.example.com/favicon/favicon-16x16.png" '
-        'rel="icon" sizes="16x16"/>'
+        'rel="icon" sizes="16x16" type="image/png">'
+    )
+    icon_32 = (
+        '<link href="_static/favicon-32x32.png" rel="icon" sizes="32x32" '
+        'type="image/png">'
     )
-    icon_32 = '<link href="_static/favicon-32x32.png" rel="icon" sizes="32x32"/>'
     icon_180 = (
         '<link href="_static/apple-touch-icon-180x180.png" '
-        'rel="apple-touch-icon" sizes="180x180"/>'
+        'rel="apple-touch-icon" sizes="180x180" type="image/png">'
     )
-
     assert icon_16 in str(index_html.select("head")[0])
     assert icon_32 in str(index_html.select("head")[0])
     assert icon_180 in str(index_html.select("head")[0])