From adff3790f3b4eb92865cdc0d6685985844351fe5 Mon Sep 17 00:00:00 2001
From: Shaun Martin
Date: Sun, 19 May 2024 21:08:35 -0500
Subject: [PATCH 1/4] WIP: add citation, prism, and dc metadata
---
Readability.js | 30 +
test/generate-testcase.js | 42 +-
test/test-pages/nature/expected-metadata.json | 10 +
test/test-pages/nature/expected.html | 702 +++++
test/test-pages/nature/source.html | 2622 +++++++++++++++++
.../ourworldindata/expected-metadata.json | 9 +
test/test-pages/ourworldindata/expected.html | 180 ++
test/test-pages/ourworldindata/source.html | 596 ++++
8 files changed, 4174 insertions(+), 17 deletions(-)
create mode 100644 test/test-pages/nature/expected-metadata.json
create mode 100644 test/test-pages/nature/expected.html
create mode 100644 test/test-pages/nature/source.html
create mode 100644 test/test-pages/ourworldindata/expected-metadata.json
create mode 100644 test/test-pages/ourworldindata/expected.html
create mode 100644 test/test-pages/ourworldindata/source.html
diff --git a/Readability.js b/Readability.js
index ee81a9be..958e9973 100644
--- a/Readability.js
+++ b/Readability.js
@@ -1629,6 +1629,14 @@ Readability.prototype = {
""
);
var parsed = JSON.parse(content);
+
+ // some sites, like ones for academic journals, separate metadata for a journal article or paper from the
+ // site's own metadata. eg: nature has only @context, @type (WebPage), and mainEntity so *all* relevant metadata
+ // would be invisible without this.
+ if (parsed["mainEntity"]) {
+ parsed = parsed["mainEntity"];
+ }
+
if (
!parsed["@context"] ||
!parsed["@context"].match(/^https?\:\/\/schema\.org\/?$/)
@@ -1729,8 +1737,12 @@ Readability.prototype = {
/\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;
// name is a single value
+<<<<<<< HEAD
var namePattern =
/^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;
+=======
+ var namePattern = /^\s*(?:(prism|citation|dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-_\.:]\s*)?(author|creator|pub-date|publicationDate|publication|description|title|site_name)\s*$/i;
+>>>>>>> 740ddd3 (WIP: add citation, prism, and dc metadata)
// Find description tags.
this._forEachNode(metaElements, function (element) {
@@ -1781,12 +1793,21 @@ Readability.prototype = {
}
// get author
+<<<<<<< HEAD
metadata.byline =
jsonld.byline ||
values["dc:creator"] ||
values["dcterm:creator"] ||
values.author ||
values["parsely-author"];
+=======
+ metadata.byline = jsonld.byline ||
+ values["dc:creator"] ||
+ values["dcterm:creator"] ||
+ values["author"] ||
+ values["parsely-author"] ||
+ values["citation_author"];
+>>>>>>> 740ddd3 (WIP: add citation, prism, and dc metadata)
// get description
metadata.excerpt =
@@ -1803,11 +1824,20 @@ Readability.prototype = {
metadata.siteName = jsonld.siteName || values["og:site_name"];
// get article published time
+<<<<<<< HEAD
metadata.publishedTime =
jsonld.datePublished ||
values["article:published_time"] ||
values["parsely-pub-date"] ||
null;
+=======
+ metadata.publishedTime = jsonld.datePublished ||
+ values["article:published_time"] ||
+ values["parsely-pub-date"] ||
+ values["citation_publication_date"] ||
+ values["prism:publicationDate"] ||
+ null;
+>>>>>>> 740ddd3 (WIP: add citation, prism, and dc metadata)
// in many sites the meta value is escaped with HTML entities,
// so here we need to unescape it
diff --git a/test/generate-testcase.js b/test/generate-testcase.js
index 70109197..82d3a477 100644
--- a/test/generate-testcase.js
+++ b/test/generate-testcase.js
@@ -1,12 +1,13 @@
/* eslint-env node, mocha */
-var debug = false;
+var debug = true;
var path = require("path");
var fs = require("fs");
var JSDOM = require("jsdom").JSDOM;
var prettyPrint = require("./utils").prettyPrint;
var http = require("http");
+var https = require("https");
var urlparse = require("url").parse;
var htmltidy = require("htmltidy2").tidy;
@@ -54,38 +55,45 @@ function generateTestcase(slug) {
});
}
-function fetchSource(url, callbackFn) {
- if (!url) {
- console.error("You should pass a URL if the source doesn't exist yet!");
- process.exit(1);
- return;
- }
- var client = http;
- if (url.indexOf("https") == 0) {
- client = require("https");
- }
+function getWithRedirects(url, cb) {
+ var client = (url.indexOf("https") == 0) ? https : http;
+
var options = urlparse(url);
options.headers = { "User-Agent": FFX_UA };
- client.get(options, function (response) {
+ client.get(options, async (response) => {
if (debug) {
console.log("STATUS:", response.statusCode);
console.log("HEADERS:", JSON.stringify(response.headers));
}
+
+ if(response.statusCode > 300 && response.statusCode <= 303) {
+ if (debug) console.log("following redirect", response.headers.location);
+ await getWithRedirects(response.headers.location, cb);
+ }
+
response.setEncoding("utf-8");
var rv = "";
response.on("data", function (chunk) {
rv += chunk;
});
- response.on("end", function () {
- if (debug) {
- console.log("End received");
- }
- sanitizeSource(rv, callbackFn);
+
+ response.on("end", function () => {
+ if (debug) console.log("End received");
+ sanitizeSource(rv, cb);
});
});
}
+function fetchSource(url, callbackFn) {
+ if (!url) {
+ console.error("You should pass a URL if the source doesn't exist yet!");
+ process.exit(1);
+ }
+
+ getWithRedirects(url, (rv) => sanitizeSource(rv, callbackFn));
+}
+
function sanitizeSource(html, callbackFn) {
htmltidy(
new JSDOM(html).serialize(),
diff --git a/test/test-pages/nature/expected-metadata.json b/test/test-pages/nature/expected-metadata.json
new file mode 100644
index 00000000..d87c2608
--- /dev/null
+++ b/test/test-pages/nature/expected-metadata.json
@@ -0,0 +1,10 @@
+{
+ "title": "Worldwide divergence of values",
+ "byline": "Medvedev, Danila",
+ "dir": null,
+ "lang": "en",
+ "excerpt": "Social scientists have long debated the nature of cultural change in a modernizing and globalizing world. Some scholars predicted that national cultures would converge by adopting social values typical of Western democracies. Others predicted that cultural differences in values would persist or even increase over time. We test these competing predictions by analyzing survey data from 1981 to 2022 (n = 406,185) from 76 national cultures. We find evidence of global value divergence. Values emphasizing tolerance and self-expression have diverged most sharply, especially between high-income Western countries and the rest of the world. We also find that countries with similar per-capita GDP levels have held similar values over the last 40 years. Over time, however, geographic proximity has emerged as an increasingly strong correlate of value similarity, indicating that values have diverged globally but converged regionally. The authors test whether social values have become converged or diverged across national cultures over the last 40 years using a 76-country analysis of the World Values Survey. They show that values have diverged, especially between high-income Western countries and the rest of the world.",
+ "siteName": "Nature",
+ "publishedTime": null,
+ "readerable": true
+}
diff --git a/test/test-pages/nature/expected.html b/test/test-pages/nature/expected.html
new file mode 100644
index 00000000..4e2133a4
--- /dev/null
+++ b/test/test-pages/nature/expected.html
@@ -0,0 +1,702 @@
+
+
+
+
Introduction
+
+
Cultural groups vary not only in their customs and traditions, but also in their values. Different national cultures place different degrees of emphasis on the collective vs. the individual1, openness vs. obedience2,3, and faith vs. skepticism4. Understanding this variation has become a central goal in the scientific study of culture, since social values shape international conflict, economic climate, and law. Studies seek to identify historical origins of cultural differences in values5,6,7 and estimate how values have changed across cultures in modern history8,9. A fundamental question for this research is whether social values are converging or diverging across cultures. Globalization and imperialism have homogenized many forms of culture, including language10 and religion11. Have values also converged across cultures in recent history, or have they remained an area of persistent or even increasing cultural divergence?
+
Traditional modernization theories famously predicted a convergence in social values. Inspired by the philosophies of Marx and Hegel, modernization theorists suggested that the end of the Cold War and the rise of globalization would catalyze the worldwide spread of a “universal civilization” with liberal and individualizing values that emphasize the primacy of personal rights and freedoms12,13,14,15. Others suggested that the global diffusion of industrialization would break down traditional culture and replace it with “modern” class structures and values16,17. These perspectives shared an assumption of unilineal modernization: that modern technology and globalization should lead world cultures to increasingly resemble democratic Western nations.
+
Scholars became more skeptical of these theories as cultural conflicts emerged across the globe throughout the 21st century. New theoretical models arose during this time, with varying levels of overlap with the older theories. Inglehart’s postmaterialist thesis suggested that globalization alone does not lead to cultural convergence, but that economic development in particular induces a shift from values prioritizing group obedience to values prioritizing self-expression and individual autonomy9,18. Welzel labeled these latter values as “emancipative,” and proposed a “human empowerment” sequence in which wealth and security encourage cultures to espouse more emancipatory values, which in turn foster participatory democracy8. Emancipative values bear a conceptual overlap and statistical correlation with individualizing values. Both emphasize the autonomy and needs of the individual over those of the group8. The post-materialist thesis therefore reproduces the assumption that cultural convergence will inherently be in the direction of Western individualism, but adds the caveat that this convergence requires economic prosperity and financial security.
+
Other theories broke further from modernization theories. Eisenstadt’s multiple modernities thesis proposed that economic development would not be a Westernizing force, but would set national cultures on unique paths towards modernization19,20. Tomlinson similarly argued that globalization might promote states in the developing world to actively develop and reinforce distinct national identities rather than Westernize21. Huntington went one step further, predicting that post-Cold War globalization would lead to a resurgence of cultural divides based on religious and linguistic differences associated with historical civilizations22.
+
Together, these theories offer a spectrum of competing hypotheses. From one perspective, high- and low-income countries should be experiencing a gradual convergence of values brought on by globalization. From another perspective, this convergence might only characterize countries that have become wealthier over time. And from the other extreme, countries should be diverging in their values, and this divergence might even be sharpest among high-income countries.
+
Evolutionary models also offer mixed predictions for value convergence. These models state that cultural differences can arise from socioecological pressures involving subsistence style23, population pressure7, resource scarcity2, climate24, and pathogen load25,26. Cultural values and norms often promote behavior that is adaptive in light of these pressures27. Sometimes these values and norms emerge over long periods of history, but they also can change quickly25,26, which evolutionary psychologists have termed “evoked culture”28,29.
+
Given the co-evolution between culture and ecology, one might expect that values should converge if people’s environments have also become more similar. In some specific ways, environments do seem to be more similar. The decline of biodiversity and the diffusion of new technology mean that people around the world consume the same foods and use the same products with greater ease than ever before21,30. However, because socioecological diversity is multidimensional and hard-to-quantify, the question of whether there has been a definitive trend toward environmental homogeneity remains open. Even if there is a trend towards environmental homogeneity, it could be affecting countries in the same regions more than countries across the globe31. Studies have found that countries in the same trading blocs have developed more similar economic, demographic, financial, and political characteristics over time, whereas countries from different trading blocs have not become more similar32.
+
The influence of Western mass media may be the most intuitive force of cultural convergence. Social learning is the dominant method through which humans transmit cultural values and norms33, and the diffusion of films, television, the internet, and educational materials have made it easier to learn about the United States that any other country34. A recent analysis found that educational attainment correlated with cultural similarity to the United States across the world, suggesting educational attainment may propagate Western values35. Yet there is less evidence that non-educational mass media is a Westernizing force. Some countries specifically ban or regulate Western media36,37. Even when foreign media is unregulated, people often prefer national and regional content38. And when people do consume foreign media, there is no clear evidence that it leads them to accept foreign values. Studies that use media as an intervention tool to change norms in non-Western cultural groups have been careful to work with local organizations to produce culture-specific productions instead of emulating Western media39,40. A mass media perspective, like an ecological perspective, offers no clear predictions for whether national cultures are converging or diverging in their values.
+
The World Values Survey (WVS) has become the proving ground for hypotheses about contemporary value change41. The WVS is a multi-panel survey of 450,869 demographically representative people across 105 countries, with multiple waves of data from 76 of these countries (406,185 people). The first timepoint (wave) of this survey took place in 1981, and the seventh timepoint completed data collection in 2021. Few studies have examined every WVS timepoint, but many have analyzed changes in the mean level of key values or value dimensions across subsets of timepoint and countries32,42,43,44,45. Some studies have argued for global trends on specific values. One analysis documented a global rise of individualism46. Another study reported that emancipative values are diffusing around the world, but this diffusion has been more rapid in liberal democracies than in other government types47. Other research has focused on specific world regions. For example, membership within the European Union (E.U.) is associated with greater value similarity with other E.U. countries over time but also with value divergence from Central Asian countries45,48. The most thorough analysis to date compared responses on three WVS questionnaires across 18 countries at two timepoints (1990s vs. 2010s)49. It reported a worldwide shift among these countries towards cultural traits typical of rich Western individualist countries, but also heterogeneity in effect sizes across the questionnaires. Together, these studies provide insights into contemporary changes in specific social values or among small samples of countries but few conclusions about worldwide trends towards value convergence vs. divergence across a large and heterogeneous pool of countries.
+
In this work, we develop a general method to test whether social values have diverged or converged across the 76 countries that have provided multiple waves of WVS data. Instead of focusing on the mean levels of particular values over time, as in most prior studies, we create metrics that explicitly measure variation in values (see Table 1). These measures are meant to represent the same outcome—convergence vs. divergence of values across world nations. However, they vary in their level of analysis. The first measure, “value variation,” focuses on divergence at the level of the WVS item whereas the second measure, “value distinctiveness,” focuses on divergence at the country level. Table 1 describes both measures and defines a third measure called “within-country heterogeneity,” which we use in secondary analyses to represent the homogeneity vs. heterogeneity of values within a nation. We define all three measures here so that readers can distinguish between them in our presentation of results.
+
+
+
+
Our main analyses test whether value variation and value distinctiveness are increasing or decreasing over time. Increases indicate value divergence, and decreases indicate value convergence. We try to test this relationship as rigorously as possible. For example, we include several robustness checks to ensure that the results are not driven by the way we normalize the items or the sampling strategy of the WVS. In our main text, we complement our primary analyses by analyzing different subsets of countries and by exploring geopolitical variables that can explain country-level clustering based on value similarity. Supplemental analyses show that results replicate with different approaches to normalizing means for items that have different response scales and across demographically weighted and unweighted country-level scores. Our materials and methods summarize our analysis procedure, and the Supplementary Methods have an “extended” materials and methods section with more detail.
+
Here we use this approach to show that values have diverged across national cultures over time. This value divergence mainly characterizes a growing gap between high-income Western countries and the rest of the world on emancipative values. We also find that worldwide value divergence has been accompanied by value convergence among countries in the same region.
+
+
+
+
Results
+
+
Value divergence at the item level
+
We first examined general trends towards value convergence or divergence using our value variation measure, which allowed us to estimate effects at the item level. Our results strongly supported value divergence. A mixed effects model with value variation nested in the 40 items found that timepoint has been significantly associated with greater value variation, b = 0.004, SE = 0.0007, t(239) = 5.12, p < 0.001, \(\beta\) = 0.17, 95% CIs [0.002, 0.005]. We replicated this result using a different approach in which we correlated timepoint with value variation separately for each of the 40 value items. Of the 40 values, we found that 27 have diverged over time, with a positive median correlation of 0.28 between timepoint and value variation, t(39) = 3.30, p = 0.002, Mdiff = 0.28, 95% CIs [0.11, 0.45]. Coefficients associated with each item are displayed in Fig. 1.
+
+
+
+
Have certain kinds of values diverged more than others? To test this question, we measured how much each item related to Welzel’s dimensions of “sacred vs. secular” values and “emancipative vs. obedient” values8,27. Welzel developed these dimensions to differentiate between values that uphold or reject tradition and religion (sacred-secular values) from those that foster or restrict the freedom of the individual from the group (emancipative-obedient values)8. We found that a broad set of items loaded on the secular-sacred dimension, ranging from the justifiability of cheating on taxes (0.46), to the confidence in churches (0.46), to the justifiability of euthanasia (0.37). A narrower set of items loaded on the emancipative-obedient dimension. Of the examples above, only justifiability of euthanasia loaded above 0.35 (0.44).
+
We found that the rate of value divergence correlated with loading on the emancipative-obedient dimension, r(38) = 0.54, p < 0.001, but not the sacred-secular dimension, r(38) = 0.19, p = 0.237. This is clearly visible in Fig. 1. The 7 items with the highest divergence scores each have high loadings on the emancipative-obedient dimension. These values were (1) justifiability of homosexuality, (2) justifiability of euthanasia, (3) importance of obedience of children, (4) justifiability of divorce, (5) justifiability of prostitution, (6) justifiability of suicide, and (7) justifiability of abortion. National cultures are diverging the most on their tolerance of individual expression versus emphasis on group obedience. Supplementary Fig. 1 reproduces Fig. 1 with shading based on secular-sacred instead of emancipative-obedient loading.
+
Our approach allowed us to understand the nature and magnitude of divergence on emancipative values. From the first to the last timepoint, value variation across countries increased 141% for justifiability of homosexuality, 94% for justifiability of prostitution, 61% for justifiability of euthanasia, and 42% for importance of childhood obedience (Fig. 2).
+
+
+
+
Divergence on emancipative values is increasingly distinguishing Western countries from non-Western ones. Consider the case of Australia and Pakistan. The first time Australians were surveyed in the WVS, 39% of participants cited childhood obedience as an important quality in children, and participants rated divorce as more unjustifiable than justifiable (0.45). When Pakistanis were first surveyed, their responses were not so different: 32% cited childhood obedience as an important childhood quality and participants rated divorce as more unjustifiable than justifiable (0.10). Over time, however, these views diverged. The last time that they were surveyed, only 18% of Australians compared to 49% of Pakistanis cited childhood obedience as an important quality, and Australians viewed divorce as much more justifiable (0.74) than Pakistanis (0.15). From the 1980s to the 2020s, similar fault lines emerged between Western and non-Western countries.
+
Rises in value variation for the 7 most divergent items are displayed in Fig. 2A. The clearest rises in value variation come from timepoints 1–5 and plateau across timepoints 5–7. Figure 2B illustrates changes over time in the means of these 7 values, which are aggregated to the continent level and coded such that higher values mean more emancipative values. This plot shows that Oceanic, European, North American, and South American countries have progressively endorsed more emancipative values, whereas endorsement of these values has been stable across Asian and African countries. Divergence on some values is driven by countries moving in opposite directions, as in the case of Australia vs. Pakistan on the value of childhood obedience (see Supplementary Table 16 for other examples). Other values diverged because their mean rate of endorsement changed in some countries but remained the same in others. We provide detailed methodological details about these items and countries in our Supplementary Information (e.g., Supplementary Tables 2–5).
+
Value divergence at the country level
+
We next replicated and extended these findings using our country-level value distinctiveness measure, in which we calculated each country’s deviation from the global median of each value. We first regressed this distinctiveness score on timepoint in a mixed effects model with observations nested in values, countries, and continents. This approach helped us account for the non-independence of countries within the same continent in our data analysis. Value distinctiveness has been rising over time, b = 0.003, SE = 0.0005, t(9290) = 4.98, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.002, 0.004], indicating that countries have diverged in their values. We replicated this effect while controlling for spatial autocorrelation more continuously using an approach similar to the one we used to calculate value distinctiveness, b = 0.003, SE = 0.0005, t(9351) = 5.18, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.002, 0.004] (see Supplementary Methods for details). This continuous method further addressed the concern that our results might have been biased by interdependence of datapoints, often called “Galton’s problem.”
+
Another concern is that these findings might be confounded with cohort effects, meaning that value divergence has resulted in changes in the WVS sampling strategy over time. If the cohorts of the WVS are becoming increasingly diverse, then the mean level of value distinctiveness would rise even if countries’ values stayed consistent over time. We took several steps to address this concern. First, we used centering to separate each country’s general value distinctiveness across all waves (representing a cohort effect) from its change over time from wave to wave (representing a longitudinal effect). A model including both variables found that the longitudinal effect was significant, b = 0.003, SE = 0.0006, t(10,520) = 4.92, p < 0.001, \(\beta\) = 0.04, 95% CIs [0.002, 0.004], but the cohort effect was not, b = 0.002, SE = 0.003, t(49.82) = 0.68, p = 0.503, \(\beta\) = 0.02, 95% CIs [−0.004, 0.009].
+
Second, we recomputed value variation and replicated our models among subsamples of countries that had less turnover and hence less susceptibility to cohort changes. We replicated value divergence among the 54 countries that participated in at least 3 waves, b = 0.003, SE = 0.0006, t(8599) = 4.82, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.002, 0.004], the 32 countries that participated in at least 4 waves, b = 0.003, SE = 0.0006, t(6145) = 4.12, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.001, 0.004], and the 18 countries that participated in at least 5 waves, b = 0.003, SE = 0.0007, t(4059) = 3.73, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.001, 0.004]. These results provided further evidence that value divergence is not an artifact of changing sampling strategy over time, but arose from national cultures changing in diverging directions over time.
+
Third, we replicated the finding in sample with no turnover in sample composition—a subset of 33 countries that provided data in the 1990s, 2000s, and 2010s, which were the three decades with the greatest WVS coverage. Value distinctiveness was higher in the 2000s, b = 0.01, SE = 0.003, t(78) = 4.05, p < 0.001, \(\beta\) = 0.29, 95% CIs [0.007, 0.02], and the 2010s, b = 0.01, SE = 0.003, t(78) = 2.66, p = 0.010, \(\beta\) = 0.19, 95% CIs [0.002, 0.02], compared to the 1990s. However, we found no significant difference between the 2000s vs. 2010s, b = −0.004, SE = 0.003, t(39) = −1.41, p = 0.167, \(\beta\) = −0.10, 95% CIs [−0.01, 0.002]. This decade-based analysis provides evidence for value divergence even when we fix the country sample over time. It also suggests that value divergence has been non-linear. In our Supplementary Methods, we evaluate different non-linear models of value divergence. These models suggest that the pace of value divergence has gradually slowed over time, rather than halting at a particular point.
+
Analyzing value distinctiveness across countries also allowed us to estimate which countries have become most dissimilar from the rest of the world. Inglehart’s post-materialist thesis suggests that high-income countries may hold especially distinctive values, at least in the domain of morality and tolerance. However, a range of other geopolitical variables could also lead cultures to adopt distinctive social values, including wealth inequality50, distance from the equator51, globalization19, and the presence of a liberal democracy47. We accessed data on these geopolitical metrics over time, and tested whether they could predict which countries were more distinctive and which countries were less distinctive. We retained the random effects structure from our previous mixed effects models to fit these results without violating any model assumptions.
+
In our regression analysis, GDP per capita was the only variable that significantly predicted value distinctiveness (see Table 2), b = 0.08, SE = 0.01, t(600.59) = 8.05, p < 0.001, \(\beta\) = 0.18, 95% CIs [0.06, 0.10], with the positive coefficient indicating that higher-income countries have more distinctive values than lower-income countries. No other predictors reached significance (ps > 0.05). In our Supplementary Table 8, we show that other geopolitical variables are not significantly linked to value distinctiveness, even when we remove GDP per capita from the model.
+
+
+
+
Further analyses found that the association between GDP per capita and value distinctiveness varied across world region. Wealth was associated with value distinctiveness among European countries, b = 0.08, SE = 0.02, t(18.76) = 3.63, p = 0.002, \(\beta\) = 0.18, 95% CIs [0.04, 0.12]. However, in a regression model where we interacted GDP per capita with continent dummy-codes, we found that, compared to Europe, the effect of GDP per capita was significantly weaker in Asia, b = −0.10, SE = 0.04, t(16.66) = −2.84, p = 0.011, \(\beta\) = −0.23, 95% CIs [−0.16, −0.04], and Africa, b = −0.21, SE = 0.10, t(122.60) = −2.13, p = 0.036, \(\beta\) = −0.50, 95% CIs [−0.40, −0.02]. The association between GDP per capita and value distinctiveness was non-significant in both Africa, b = −0.14, SE = 0.10, t(109.20) = −1.38, p = 0.171, \(\beta\) = −0.32, 95% CIs [−0.32, 0.06], and Asia, b = −0.02, SE = 0.03, t(16.24) = −0.78, p = 0.448, \(\beta\) = −0.05, 95% CIs [−0.08, 0.03]. These continent comparison models were based on smaller subsets of countries, with 48 countries in the Europe vs. Asia comparison (simulated power = 80.20%) and 36 countries in the Europe vs. Africa comparison (simulated power = 98.00%). Readers should therefore interpret these findings with more caution than the tests that included the full sample of countries. Supplementary Table 13 summarizes the complete set of coefficients for these models. Our Supplementary Methods provide more details about the power analysis simulations.
+
Figure 3 illustrates many of the findings that we have presented in a single plot, with countries ordered along the y-axis based on their continent and on their level of value distinctiveness within continents. The x-axis illustrates each country’s value distinctiveness score at the first WVS timepoint and the final WVS timepoint, and each country’s GDP per capita is shown on the left of the y-axis. In this visualization, it is clear that most countries have diverged from the rest of the world, with more distinctive values at the final vs. the first timepoint. Second, in Western regions but not non-Western regions, divergence has been most stark for high-income countries.
+
+
+
+
These findings suggest a provocative possibility: rises in global wealth may be responsible for the worldwide divergence of values. Most countries have become wealthier over time—Western countries but also many non-Western countries like Singapore, Hong Kong, and South Korea52. This means that most countries were poorer in the earlier (vs. later) waves of the WVS. Western nations in these early waves held more emancipative values than non-Western nations, but not by a large degree. As time passed, rising wealth led Western countries to adopt more emancipative values, but it did not have the same effect for most non-Western countries. These trends led to a growing gap between high-income Western countries and the rest of the world.
+
Consider Hong Kong and Canada, where GDP per capita has followed a similar trend, but values have diverged. Both countries had a GDP per capita of approximately $25,000 in 2000, which doubled to approximately $50,000 by 202053,54. Over the same time interval, beliefs that homosexuality was justifiable rose in Canada from 0.49/1.00 to 0.74/1.00. Perceived justifiability of homosexuality also rose in Hong Kong, but only from 0.29/1.00 to 0.44/1.00. This means that the gap in means between Hong Kong and Canada increased by 50% during this period. One of the fastest-changing values in Hong Kong during this time was belief in children’s work ethic. From 2000 to 2021, the percent of Hong Kong participants who mentioned responsibility as an important childhood quality rose from 19% to 52%, whereas it fell from 53% to 47% in Canada.
+
This example shows that wealth can sometimes lead to region-specific effects on values. If rising wealth fosters emancipative values in some regions but not in others, this could explain why Western democracies have developed more distinct values over time. This would be consistent with Eisenstadt, whose “multiple modernities” thesis states that countries follow their own trajectories of modernization19. It is also consistent with Huntington’s observations that rising wealth and influence in East Asia could lead to a re-affirmation of traditional Confucian values22. Factors such as migration, political change, and decolonialization could also contribute to these trends. For example, sovereignty over Hong Kong transitioned from the United Kingdom to China in 1997, which may have affected people’s values. More research is necessary to determine the causal relationship, if any, between wealth and value change.
+
Within-country heterogeneity and value distinctiveness
+
In our next analysis, we explored whether value distinctiveness across countries was correlated with the heterogeneity of values within countries. In other words, are countries where citizens disagree on values more like the rest of the world than countries where citizens have more homogeneous values?
+
We calculated within-country heterogeneity using the same approach that we used to calculate value distinctiveness. But instead of taking the difference of country means from the global mean across values, we took the deviation of individual ratings from the country mean. Higher levels of within-country heterogeneity therefore represent a country with high levels of disagreement among its citizens. For example, high within-country heterogeneity in the United States might arise because liberal and conservative Americans hold very different values. In exploratory analyses, we analyzed whether within-country heterogeneity was also rising across time, signaling a divergence within countries. However, we did not find general trends in this measure. Within-country heterogeneity seems to be rising among some countries and falling among others (see Supplementary Methods).
+
However, we did find that within-country heterogeneity was closely linked to value distinctiveness. When we regressed value distinctiveness on within-country heterogeneity controlling for GDP per capita, Gini, globalization, political rights, and distance from equator, within-country heterogeneity robustly and negatively predicted value distinctiveness, b = −0.06, SE = 0.01, t(8,032) = −4.45, p < 0.001, \(\beta\) = −0.07, 95% CIs [−0.09, −0.03]. In other words, countries where citizens agree more on values tend to be more different from other countries. The other effects in this model remained unchanged. GDP per capita had a positive relationship and no other variables had significant associations (see Supplementary Table 12). Figure 4 visualizes the relationship between within-country heterogeneity and value distinctiveness. Our Supplementary Methods reports timepoint specific models.
+
+
+
+
These results suggest that as countries develop more homogenous values, they may increasingly splinter from viewpoints that are shared around the rest of the world. This has happened in high-income Western nations like Sweden and Norway, where there is a higher consensus around emancipative values such as justifiability of divorce and abortion. But it has also happened in poorer nations such as China and Ghana, where consensus has formed on different values. In contrast, countries with heterogeneous values such as South Africa and India may struggle with internal division, but when averaging across their subpopulations, these countries’ values may resemble those of other countries.
+
What country characteristics predict value similarity over time?
+
In our final analysis, we explored which countries hold similar values to one another. We first fit a principal components analysis (PCA) to detect underlying dimensions in how the 40 values varied across nations. Two principal components (PCs) explained 80.70% of variation (see Materials and Methods). To understand the meaning of these dimensions, we regressed each dimension against previously developed dimensions of value variation: Welzel’s secular values and emancipative values, and Inglehart’s 12-item post-materialist index. PC1 was strongly positively linked to emancipative values, b = 0.76, SE = 0.04, t(215.60) = 19.46, p < 0.001, \(\beta\) = 0.77, 95% CIs [0.67, 0.84], and to secular values, b = 0.26, SE = 0.03, t(233.90) = 8.60, p < 0.001, \(\beta\) = 0.26, 95% CIs [0.20, 0.32]. PC2 had a smaller and less robust relationship with secular values, b = −0.13, SE = 0.07, t(194.81) = −2.10, p = 0.042, \(\beta\) = −0.13, 95% CIs [−0.27, 0.009]. No effects involving the choice index were robust (see Supplementary Methods for more information about these models).
+
We used these PCs as coordinates in a two-dimensional value space in which countries with similar values are closer together than nations with different values (see Fig. 5). We next fit regression models for each timepoint to test whether geopolitical characteristics could explain which countries were closer together in value space and which countries were further apart.
+
+
+
+
These regressions found that wealth has been the strongest correlate of value similarity across nations over time. High-income countries share values with other high-income countries, and poor countries share values with poor countries. This effect is visible in Fig. 5 (middle right), where high-income countries like New Zealand, Germany, the Netherlands, and Japan are clustered together on the right side of the value space. We also note that the left side of the value space in Fig. 5 is denser than the right side, which replicates our prior analysis: high-income countries have developed more dissimilar values compared to the rest of the world, primarily because of their unique endorsement of emancipative values.
+
We also found that geographic proximity has become progressively more correlated with value similarity over time, with an effect size rising from 0.16 (Timepoint 1) to 0.35 (Timepoint 7). This is visible in Fig. 5A: high-income East Asian countries like South Korea and Singapore are closer together than high-income Western nations like New Zealand and the Netherlands. The rising predictive power of geographic proximity suggests that values are converging within regions but are diverging across regions.
+
Religion has also emerged as a robust predictor of value similarity. Countries with more similar religious profiles have more similar values, even controlling for their similarity in wealth, geographic position, and other geopolitical features. This finding conceptually replicates a recent analysis that found that co-religionists—even those living in similar countries—shared similar values55. The fact that values are segregating along geographical and religious fault-lines further supports Huntington’s thesis that the 21st century would see a rise of ancestral cleavages in values. By the final WVS timepoint, religious similarity and geographic proximity were stronger correlates of value similarity than having similar levels of inequality or political rights. Figure 5 plots out the associations of value similarity with each form of geopolitical similarity at each timepoint. Table 3 lists key coefficients from the value similarity regressions.
+
+
+
+
+
+
+
Discussion
+
+
We find that values have diverged from 1981 to 2022 across 76 countries. Value divergence appears to be strongest for values related to tolerance and openness and can be explained by the rising difference between high-income Western countries and the rest of the world. Regional value convergence has accompanied worldwide divergence, with geographically proximal countries adopting more similar values over time.
+
Our Supplementary Methods present many robustness tests that replicate our key findings with different sampling strategies, normalization procedures, and demographic weighting. They also include an additional literature review that summarizes previous research on modern cultural differences in values and other metrics of cultural distance in greater depth.
+
These robustness analyses lend more support to our conclusions, but there are still important limitations to this study. First, we can only claim that values have diverged across the 40 items in our analysis. These items include a wide range of different values. However, they are not an exhaustive list of values across countries, and our results could change if they were replicated on a different set of items. Second, our samples are not perfectly representative of populations in the countries that we have analyzed. The WVS attempts to emulate key dimensions of representativeness, but it may sometimes neglect responses from key demographics, such as indigenous populations. For these reasons, it will be important to replicate our analyses in new datasets. Few datasets have data on social values that match the WVS, but this may change as new cross-cultural surveys are launched and analyzed over time.
+
While acknowledging these limitations, we also view our findings as potentially impactful from both a theoretical and practical standpoint. Theoretically, our findings suggest that globalization and intergroup contact alone are not sufficient to produce converging social values. Our findings also suggest that the post-materialist hypothesis—that wealth breeds emancipative values and tolerance8,9—may have stronger predictive power in some regions than others. Our findings support key parts of other theories but do not completely align with any single theory of culture and values. We observe worldwide divergence of values accompanied by re-alignment of values along regional and religious fault-lines, which is consistent with Huntington’s civilizations thesis and more recent work on rises in geopolitical regionalism from macroeconomics56. We also find that this effect of wealth varies across Western and non-Western countries, consistent with Eisenstadt’s multiple modernities theory and with other studies that highlight Asia’s unique modernization trajectory57. It may be that our findings are specific to a particular period of time following decolonization and the end of the Cold War (1981–2022) and that we would have found different results at different periods of time. Only time will tell if our findings represent a general cultural trend or a historically isolated phenomenon.
+
Value divergence could also explain theoretical puzzles in the social sciences. For example, there is a popular theory that rising wealth4 and technology58 facilitate religious decline because they decrease existential insecurity and relieve the economic pressure to have children59. But this model does not explain why rising wealth has not brought religious declines in Middle Eastern countries and has even correlated with rising religiosity in some of these countries. One possibility is that these declines will happen with time, especially with generational change. In other words, countries like Saudi Arabia and Qatar may not have been high-income long enough to experience secularization. However, another possibility is that Islam has emerged as a key part of Arabic post-colonial identity. Rising wealth and influence could have led Arab countries to emphasize the religious part of their identity to distinguish themselves from the West.
+
Our findings answer some important questions about contemporary value change, but they also pose new ones. Why has rising wealth led Western, but not non-Western, nations to espouse more emancipative values? We believe that this relationship might be embedded in ecological and cultural events throughout Western history, including the advent of participatory democracy in Athens and early Rome, enlightenment and post-enlightenment philosophy, the Catholic Church’s marriage and family polices60, the French revolution, and the reformation61. These phenomena may have gradually solidified a Western identity focused on autonomy, primacy of individual rights over obligations to the in-group, and tolerance for breaking norms2,60,62. As the world has globalized and Western nations have competed for resources on the world stage, this identity may have crystalized even further. But this does not mean that wealth or globalization should have encouraged similar values in African, East Asian, or South Asian cultures. To the contrary, growing power and resources may have prompted non-Western countries to affirm their own traditional values.
+
Practically, value divergence has implications for political polarization and conflict across world countries. Russia has framed the recent war in Ukraine as a war against Western values40. Chinese politicians have spoken against countries that “forcibly promote the concept and system of Western democracy and human rights”63. Western non-governmental organizations have faced recent accusations of seeding immorality and propagating Western imperialism64, and public opinion polling has found increasingly hostile attitudes towards Western countries in the Middle East65, Asia66, and Africa67. Our findings do not shed light on the extent of the anti-Western sentiment or the exact nature of its antecedents and consequences. We do not know to what extent governments strategically propagate certain values to reinforce national identities, as Tomlinson21 would suggest. One goal of future research could be to understand the extent that political elites have encouraged value divergence among ordinary citizens.
+
Our research also underscores the limitations of studying Westerners to make claims about human psychology writ large. Cross-cultural scholars have pointed out that people from Western, Educated, Industrialized, Rich, and Democratic (WEIRD) countries have psychological traits that differ from the rest of the world. This peculiarity presents an external validity problem for studies that recruit mostly WEIRD subjects, and it also presents an intellectual problem for cultural evolutionists who hope to explain regional variation. We show that this problem has become more acute in the last forty years. WEIRD subjects have become even more peculiar, at least in their social and moral values. This shift makes it more crucial than ever that behavioral scientists develop their theories using data from globally representative samples.
+
+
+
+
Methods
+
+
Here we report key parts of our methodology and analysis plan. Our Supplementary Methods contain an extended materials and methods section with further information.
+
Ethics
+
The director of the Social and Behavioral Science IRB office at University of Chicago determined that our research does not require IRB approval, since the nature of this research—analyzing archival deidentified data—does not quality as human subjects research.
+
The World Values Survey
+
The World Values Survey (WVS) is an international research program devoted to measuring the social, political, economic, and religious values of individuals around the world using regular surveys. The WVS website contains comprehensive information about its research procedures (https://www.worldvaluessurvey.org). This includes information about translation procedures and fieldwork training. In addition to publishing data each wave, the WVS publishes a time-series file containing data from all waves. The WVS has not surveyed the same people over time in this file. Rather, each timepoint contains a demographically representative snapshot of people in a country at a particular point in time. The WVS also publishes a list of variables indicating which items are asked in different waves, and a list of countries indicating which countries are surveyed in each wave. The timeseries dataset is published in many different formats. We downloaded the Rdata format.
+
Characteristics of countries
+
We focused on the 76 countries where the WVS has collected data for at least two waves, a necessary condition since we are interested in change over time. Supplementary Table 1 summarizes the age-sex composition of each country in each wave, with dashes indicating waves where a country was not included in data collection. Samples are designed to be representative of people age 18 and older residing within private residences in each country, regardless of citizenship or language. The WVS employs probability sampling and stratified sampling to achieve these targets. They offer case weights to compensate for small deviations with respect to gender-age (self-reported), rural-urban, or educational attainment.
+
Characteristics of Items
+
The WVS contains a heterogeneous set of items that change each year. Of the 1,011 items included in the WVS timeseries file, most were only included in a minority of waves. We focused on the items that were asked in all seven waves, and specifically on items that could be construed broadly as values. In total, we selected 40 items for analysis. Our extended Supplementary Methods summarize how we selected these items and excluded others. In Supplementary Table 2, we provide the item identification number, item label, and the scale that participants used to respond to the item. Readers can access the complete item wording for each item by downloading one of the PDF questionnaires for any wave from the WVS website.
+
One of the scales in our analysis involved the qualities that respondents felt to be important for children to learn. For these items, respondents were not allowed to select more than 5 items. However, during data processing we realized that this rule was not always followed. To keep the questionnaire format consistent across countries and waves, we excluded all respondents (n = 20,380; 5% of the total sample) who had selected more than 5 important childhood qualities. This decision did not affect our results. All results replicated with and without excluding participants who had not followed the instructions.
+
Exogenous variables
+
We collected data on exogenous variables illustrating the geopolitical conditions of countries over time. We sought to match all exogenous variables as closely as possible to the year of WVS data collection, so we downloaded time-varying estimates of these variables. We provide the source and access URL for each variable here. Our extended materials and methods provide more information about each variable.
We measured political rights using the “Political Rights” index published by the Freedom House. We accessed this variable using the “Global Economy” database. The access URL is https://www.theglobaleconomy.com/download-data.php. We note that the Freedom House publishes two different indices: a political rights index and a civil liberties index. Both indices are coded on a 7-point scale from 1 (Strong) to 7 (Weak). These correlated at 0.94 and showed identical results, so we focused on the political rights index in our analyses. We recoded the index so that higher values meant more political rights.
+
Religious distance
+
We accessed data on the % of the population identifying as Hindu, Christian, Muslim, and Buddhist, and measured religious distance by summing the absolute difference in % of people identifying with each religious group across pairs of countries. The access URL is https://www.theglobaleconomy.com/download-data.php.
+
Analytic strategy
+
This section summarizes how we computed value variation and value distinctiveness. Table 1 summarizes the meaning of these measures, along with within-country heterogeneity, which we describe in the main text.
+
Item normalization
+
Supplementary Table 2 shows that items were asked on different scales. Some items involved binary responses (e.g., whether people mention not wanting to be neighbors with someone from a specific group). Others were asked with Likert-type scales, such as the 1–10 scale that people used to rate whether behaviors were morally justifiable. We normalized the item scales using min-max normalization, which is a common approach in data science and machine learning68. Given a vector V = [v1, v2, … vn], we can determine min_V as the minimum value in the vector and max_V as the maximum value in the vector. We can create our normalized vector, V’ using:
In other words, each element in the new vector is the result of subtracting the minimum value of the original vector from that element, then dividing by the range of the original vector (i.e., the difference between its maximum and minimum values). This procedure results in variables with ranges of 0–1, no matter their original scales. As an alternative to min-max normalization, we also considered a median split approach. We discuss the limitations of this approach in the Supplementary Methods, and show how our main findings replicate with this approach.
+
Calculating value divergence across items
+
After the item-normalization procedure, we took the mean values of items for each country. This resulted in country-level means for each item at each timepoint. As an exploratory analysis, we fit Pearson correlations between timepoint and mean for each item, which indicates how values are changing over time across all countries. Supplementary Table 3 lists the full set of trends in global mean endorsement of items over time.
+
We assessed value divergence across items by computing the standard deviation of country means at each WVS timepoint—the measure of “value variation” that we summarize in our introduction. We then estimated the linear trend in these SD values in a mixed effects model which we summarize in the main text, and also across a set of Pearson correlations that we fit for each individual item and that we summarize in Fig. 1. In these models, positive coefficients represent value divergence, since the SD of country means is increasing over time. In Supplementary Table 4, we report value divergence coefficients for four different subsets of countries— countries which were included in at least 2 WVS waves, 3 waves, 4 waves, and 5 waves. We conducted our analyses for these different subsets of countries because we wanted to ensure that value divergence was not due to changing WVS composition over time.
+
We calculated the relevance of each item to Welzel’s emancipative and secular dimensions by taking the absolute value of the correlation between the item and each of the two dimensions to create continuous loading scores. Scores for both dimensions are published as variables in the WVS dataset. The item measuring participants’ view of “unselfishness” as an important child quality had a loading of 0.01 on the emancipative values dimension and a loading of 0.003 on the secular values dimension, suggesting it had low relevance for both dimensions. In contrast, the item measuring participants’ views on the justifiability of had a loading of 0.56 on the emancipative factor and a loading of 0.40 on the secular factor, reflecting higher relevance for both factors. The dimensions correlate at r = 0.36 with each other, suggesting that secular values tend to be more emancipative and that the two dimensions are not redundant.
+
Calculating value distinctiveness
+
We followed a multi-step process to calculate value distinctiveness for each country. First, we computed the global median score for each value at each timepoint. For example, the median score for the item “Important Child Qualities: Hard Work” was 0.55 in the third WVS wave. Next, we computed the absolute differences between each country’s mean and the global median for each item. For example, the average response to the “Hard Work” item in Albania was 0.57, yielding an absolute difference of 0.02. Finally, we aggregated across all of these absolute differences to obtain a “value distinctiveness” score for each country. This process of computing value distinctiveness Di,j can be expressed as:
Where ni,j represents the mean value j of a given country i. We used the median to compute the global value because it avoided outlier countries from having a large impact on the mean value.
+
Computing value distinctiveness across timepoints allowed us to estimate which countries have relatively unique values at any given time. In our main text, Fig. 3 summarizes each country’s value distinctiveness score in the first and last wave that it was included in the WVS. Supplementary Fig. 2 visualizes value distinctiveness for each country in each WVS wave, and Supplementary Table 5 summarizes value distinctiveness for each country in each wave.
+
In addition to estimating the value distinctiveness of individual countries at specific timepoints, we also analyzed general trends in value distinctiveness over time. Our reasoning was that, if countries are diverging in their values, the average value distinctiveness coefficient would be increasing. This would indicate that countries are “spreading out” around the global medians of values. One limitation of this analysis is that the global midpoint is not truly “global”—it is only the average of the countries sampled by the WVS at a particular point in time. If the WVS has become systematically more diverse in its sampling, then this could artificially create value divergence via a trend in sample heterogeneity. This is why we repeated all of our analyses for subsets of countries that had participated in 2, 3, 4, and 5 WVS waves. It is also why we conducted the decade-over-decade analysis in which we replicated the finding when looking across a subset of 33 countries which provided data in the 1990s, 2000s, and 2010s—the three decades with the greatest WVS coverage. Supplementary Table 6 summarizes value distinctiveness scores for each decade and for each country within the 33-country sample.
+
Clustering methodology
+
In our final analysis, we sought to project countries onto an n-dimensional value space. Greater distance in this space between a pair of countries would represent a larger difference between the values of two countries. We could also test which geopolitical variables were most strongly correlated with this pairwise distance metric.
+
The first step in this process was to determine how many dimensions would appropriately capture sufficient variance across values. Supplementary Fig. 3 illustrates the correlations between all 40 items in our dataset. There were clear covariances across sets of values, and so we reasoned that a small number of dimensions might explain considerable variation across values. To determine the optimal number of dimensions, we fit a Principal Components Analysis (PCA) on the correlation matrix of values (the same matrix displayed in Supplementary Fig. 3). In this analysis, PC1 explained 65.4% of variation, and PC2 explained 15.3% of variation. No other dimension explained more than 10% of variation, so we adopted a 2-dimension solution. Supplementary Fig. 3 shows an elbow plot of variance explained by each PC, and the item loadings. We projected countries onto a two-dimensional value space using these PC item loadings multiplied by the country’s scores on each value. This procedure generated the plots displayed in Fig. 5.
+
The regression that we describe in our main text (see Table 2) was fit to a dataframe where rows represented pairs of nations, with columns indicating each nation in the pairwise comparison. Subsequent columns indicated Euclidean distance in the two-dimensional value space, difference in GDP per capita, geographical distance between nations, etc. We fit a cross-classified mixed effects model with random effects for each nation in the pairwise comparison, and fixed effects entered simultaneously to control for covariation between geopolitical characteristics. We standardized all variables in the regression so that estimates were not influenced by the original measurement scales of the geopolitical variables. More positive values in these fixed effects suggest that countries with similar geopolitical characteristics also have more similar values. For example, a positive effect of GDP per capita would suggest that countries with very similar levels of wealth also have very similar social values. Changes over time in these coefficients would indicate that a geopolitical variable is becoming better or worse at predicting which countries had more similar values.
All data are available on the open science framework at https://doi.org/10.17605/OSF.IO/F9BZ7. Data on values can also be publicly downloaded from the World Values Survey at https://www.worldvaluessurvey.org/. Data on GDP per capita, annual GDP per capita growth, and the Gini coefficient were retrieved from the World Bank (https://data.worldbank.org/). Data on Gini coefficients was retrieved from the World Inequality Database (https://wid.world/data/). We retrieved data on globalization from the Swiss Economic Institute and data on political rights from the Freedom House. We used theGlobalEconomy.com, a dataset aggregator and supplier, to retrieve both datasets (https://www.theglobaleconomy.com/download-data.php). We also used theGlobalEconomy.com to retrieve our data on religious distance using the same link.
Norris, P. & Inglehart, R. Sacred and secular: Religion and politics worldwide. (Cambridge University Press, 2011).
+
+
+
Schulz, J. F., Bahrami-Rad, D., Beauchamp, J. P. & Henrich, J. The Church, intensive kinship, and global psychological variation. Science366, eaau5141 (2019).
Fukuyama, F. The end of history and the last man. (Simon and Schuster, 2006).
+
+
+
Fukuyama, F. The origins of political order: From prehuman times to the French Revolution. (Farrar, Straus and Giroux, 2011).
+
+
+
Giddens, A. Time, space and regionalisation. In Social relations and spatial structures. (eds. Gregory, D. & Urry, J.) 265–295 (St. Martin’s, New York, 1985).
+
+
+
Naipaul, V. S. Beyond belief: Islamic excursions among the converted peoples. (Pan Macmillan, 2012).
+
+
+
Kerr, C., Dunlop, J. T., Harbison, F. H. & Myers, C. A. Postscript to Industrialism and industrial man. Int. Lab Rev.103, 519–540 (1971).
Eisenstadt, S. N. Multiple modernities in an age of globalization. Grenzenlose Ges. Verhandlungen 29 Kongresses Dtsch. Ges. Für Soziol. 16 Kongresses Österr. Ges. Für Soziol. 11 Kongresses Schweiz. Ges. Für Soziol. Freibg. Br 1998 37–50 (1999).
+
+
+
Eisenstadt, S. N., Riedel, J. & Sachsenmaier, D. The context of the multiple modernities paradigm. in Reflections on multiple modernities 1–23 (Brill, 2002).
+
+
+
Tomlinson, J. Globalization and culture. (University of Chicago Press, 1999).
+
+
+
Huntington, S. P. The clash of civilizations? (Springer, 2000).
+
+
+
Thomson, R. et al. Relational mobility predicts social behaviors in 39 countries and is tied to historical farming and threat. Proc. Natl Acad. Sci.115, 7521–7526 (2018).
Grossmann, I. & Varnum, M. E. Social structure, infectious diseases, disasters, secularism, and cultural change in America. Psychol. Sci.26, 311–324 (2015).
White, C. & Muthukrishna, M. More educated, more WEIRD: Higher education predicts worldwide cultural similarity to the United States and other WEIRD countries. https://doi.org/10.31234/osf.io/8wr5d (2023).
Andersen, R. & Curtis, J. Social class, economic inequality, and the convergence of policy preferences: Evidence from 24 modern democracies. Can. Rev. Sociol. Can. Sociol.52, 266–288 (2015).
Roberts, L. L. Changing worldwide attitudes toward homosexuality: The influence of global and region-specific cultures, 1981–2012. Soc. Sci. Res.80, 114–131 (2019).
Inglehart, R. C., Nash, R., Hassan, Q. N. & Schwartzbaum, J. Attitudes toward euthanasia: a longitudinal analysis of the role of economic, cultural, and health-related factors. J. Pain. Symptom. Manag.62, 559–569 (2021).
Akaliyski, P. & Welzel, C. Clashing Values: Supranational Identities, Geopolitical Rivalry and Europe’s Growing Cultural Divide. J. Cross-Cult. Psychol.51, 740–762 (2020).
Van de Vliert, E. & Van Lange, P. A. Latitudinal psychology: An ecological perspective on creativity, aggression, happiness, and beyond. Perspect. Psychol. Sci.14, 860–884 (2019).
White, C. J., Muthukrishna, M. & Norenzayan, A. Cultural similarity among coreligionists within and between countries. Proc. Natl. Acad. Sci. 118, e2109650118 (2021).
+
+
+
Pomfret, R. Is regionalism an increasing feature of the world economy? World Econ.30, 923–947 (2007).
Bomhoff, E. J. & Gu, M. M.-L. East Asia Remains Different: A Comment on the Index of “Self-Expression Values,” by Inglehart and Welzel. J. Cross-Cult. Psychol.43, 373–383 (2012).
Jackson, J. C., Yam, K. C., Tang, P. M., Sibley, C. G. & Waytz, A. Exposure to automation explains religious declines. Proc. Natl Acad. Sci.120, e2304748120 (2023).
We thank Rick Shweder, Eli Finkel, Michele Gelfand, Nour Kteily, and Nava Caluori for comments on an earlier draft of this paper. We thank the World Values Survey for providing high-quality, open-access time-varying data on values around the world.
+
+
+
Author information
+
+
Authors and Affiliations
+
+
+
Booth School of Business, University of Chicago, Chicago, USA
+ Nature Communications thanks Michael Varnum and the other, anonymous, reviewer(s) for their contribution to the peer review of this work. A peer review file is available.
+
+
+
+
+
Additional information
+
+ Publisher’s note Springer Nature remains neutral with regard to jurisdictional claims in published maps and institutional affiliations.
+
+
+
+
Supplementary information
+
+
+
Rights and permissions
+
+
+ Open Access This article is licensed under a Creative Commons Attribution 4.0 International License, which permits use, sharing, adaptation, distribution and reproduction in any medium or format, as long as you give appropriate credit to the original author(s) and the source, provide a link to the Creative Commons licence, and indicate if changes were made. The images or other third party material in this article are included in the article’s Creative Commons licence, unless indicated otherwise in a credit line to the material. If material is not included in the article’s Creative Commons licence and your intended use is not permitted by statutory regulation or exceeds the permitted use, you will need to obtain permission directly from the copyright holder. To view a copy of this licence, visit http://creativecommons.org/licenses/by/4.0/.
+
+ Thank you for visiting nature.com. You are using a browser version with limited support for CSS. To obtain the best experience, we recommend you use a more up to date browser (or turn off compatibility mode in Internet Explorer). In the meantime, to ensure continued support, we are displaying the site without styles and JavaScript.
+
+ Social scientists have long debated the nature of cultural change in a modernizing and globalizing world. Some scholars predicted that national cultures would converge by adopting social values typical of Western democracies. Others predicted that cultural differences in values would persist or even increase over time. We test these competing predictions by analyzing survey data from 1981 to 2022 (n = 406,185) from 76 national cultures. We find evidence of global value divergence. Values emphasizing tolerance and self-expression have diverged most sharply, especially between high-income Western countries and the rest of the world. We also find that countries with similar per-capita GDP levels have held similar values over the last 40 years. Over time, however, geographic proximity has emerged as an increasingly strong correlate of value similarity, indicating that values have diverged globally but converged regionally.
+
+ Cultural groups vary not only in their customs and traditions, but also in their values. Different national cultures place different degrees of emphasis on the collective vs. the individual1, openness vs. obedience2,3, and faith vs. skepticism4. Understanding this variation has become a central goal in the scientific study of culture, since social values shape international conflict, economic climate, and law. Studies seek to identify historical origins of cultural differences in values5,6,7 and estimate how values have changed across cultures in modern history8,9. A fundamental question for this research is whether social values are converging or diverging across cultures. Globalization and imperialism have homogenized many forms of culture, including language10 and religion11. Have values also converged across cultures in recent history, or have they remained an area of persistent or even increasing cultural divergence?
+
+
+ Traditional modernization theories famously predicted a convergence in social values. Inspired by the philosophies of Marx and Hegel, modernization theorists suggested that the end of the Cold War and the rise of globalization would catalyze the worldwide spread of a “universal civilization” with liberal and individualizing values that emphasize the primacy of personal rights and freedoms12,13,14,15. Others suggested that the global diffusion of industrialization would break down traditional culture and replace it with “modern” class structures and values16,17. These perspectives shared an assumption of unilineal modernization: that modern technology and globalization should lead world cultures to increasingly resemble democratic Western nations.
+
+
+ Scholars became more skeptical of these theories as cultural conflicts emerged across the globe throughout the 21st century. New theoretical models arose during this time, with varying levels of overlap with the older theories. Inglehart’s postmaterialist thesis suggested that globalization alone does not lead to cultural convergence, but that economic development in particular induces a shift from values prioritizing group obedience to values prioritizing self-expression and individual autonomy9,18. Welzel labeled these latter values as “emancipative,” and proposed a “human empowerment” sequence in which wealth and security encourage cultures to espouse more emancipatory values, which in turn foster participatory democracy8. Emancipative values bear a conceptual overlap and statistical correlation with individualizing values. Both emphasize the autonomy and needs of the individual over those of the group8. The post-materialist thesis therefore reproduces the assumption that cultural convergence will inherently be in the direction of Western individualism, but adds the caveat that this convergence requires economic prosperity and financial security.
+
+
+ Other theories broke further from modernization theories. Eisenstadt’s multiple modernities thesis proposed that economic development would not be a Westernizing force, but would set national cultures on unique paths towards modernization19,20. Tomlinson similarly argued that globalization might promote states in the developing world to actively develop and reinforce distinct national identities rather than Westernize21. Huntington went one step further, predicting that post-Cold War globalization would lead to a resurgence of cultural divides based on religious and linguistic differences associated with historical civilizations22.
+
+
+ Together, these theories offer a spectrum of competing hypotheses. From one perspective, high- and low-income countries should be experiencing a gradual convergence of values brought on by globalization. From another perspective, this convergence might only characterize countries that have become wealthier over time. And from the other extreme, countries should be diverging in their values, and this divergence might even be sharpest among high-income countries.
+
+
+ Evolutionary models also offer mixed predictions for value convergence. These models state that cultural differences can arise from socioecological pressures involving subsistence style23, population pressure7, resource scarcity2, climate24, and pathogen load25,26. Cultural values and norms often promote behavior that is adaptive in light of these pressures27. Sometimes these values and norms emerge over long periods of history, but they also can change quickly25,26, which evolutionary psychologists have termed “evoked culture”28,29.
+
+
+ Given the co-evolution between culture and ecology, one might expect that values should converge if people’s environments have also become more similar. In some specific ways, environments do seem to be more similar. The decline of biodiversity and the diffusion of new technology mean that people around the world consume the same foods and use the same products with greater ease than ever before21,30. However, because socioecological diversity is multidimensional and hard-to-quantify, the question of whether there has been a definitive trend toward environmental homogeneity remains open. Even if there is a trend towards environmental homogeneity, it could be affecting countries in the same regions more than countries across the globe31. Studies have found that countries in the same trading blocs have developed more similar economic, demographic, financial, and political characteristics over time, whereas countries from different trading blocs have not become more similar32.
+
+
+ The influence of Western mass media may be the most intuitive force of cultural convergence. Social learning is the dominant method through which humans transmit cultural values and norms33, and the diffusion of films, television, the internet, and educational materials have made it easier to learn about the United States that any other country34. A recent analysis found that educational attainment correlated with cultural similarity to the United States across the world, suggesting educational attainment may propagate Western values35. Yet there is less evidence that non-educational mass media is a Westernizing force. Some countries specifically ban or regulate Western media36,37. Even when foreign media is unregulated, people often prefer national and regional content38. And when people do consume foreign media, there is no clear evidence that it leads them to accept foreign values. Studies that use media as an intervention tool to change norms in non-Western cultural groups have been careful to work with local organizations to produce culture-specific productions instead of emulating Western media39,40. A mass media perspective, like an ecological perspective, offers no clear predictions for whether national cultures are converging or diverging in their values.
+
+
+ The World Values Survey (WVS) has become the proving ground for hypotheses about contemporary value change41. The WVS is a multi-panel survey of 450,869 demographically representative people across 105 countries, with multiple waves of data from 76 of these countries (406,185 people). The first timepoint (wave) of this survey took place in 1981, and the seventh timepoint completed data collection in 2021. Few studies have examined every WVS timepoint, but many have analyzed changes in the mean level of key values or value dimensions across subsets of timepoint and countries32,42,43,44,45. Some studies have argued for global trends on specific values. One analysis documented a global rise of individualism46. Another study reported that emancipative values are diffusing around the world, but this diffusion has been more rapid in liberal democracies than in other government types47. Other research has focused on specific world regions. For example, membership within the European Union (E.U.) is associated with greater value similarity with other E.U. countries over time but also with value divergence from Central Asian countries45,48. The most thorough analysis to date compared responses on three WVS questionnaires across 18 countries at two timepoints (1990s vs. 2010s)49. It reported a worldwide shift among these countries towards cultural traits typical of rich Western individualist countries, but also heterogeneity in effect sizes across the questionnaires. Together, these studies provide insights into contemporary changes in specific social values or among small samples of countries but few conclusions about worldwide trends towards value convergence vs. divergence across a large and heterogeneous pool of countries.
+
+
+ In this work, we develop a general method to test whether social values have diverged or converged across the 76 countries that have provided multiple waves of WVS data. Instead of focusing on the mean levels of particular values over time, as in most prior studies, we create metrics that explicitly measure variation in values (see Table 1). These measures are meant to represent the same outcome—convergence vs. divergence of values across world nations. However, they vary in their level of analysis. The first measure, “value variation,” focuses on divergence at the level of the WVS item whereas the second measure, “value distinctiveness,” focuses on divergence at the country level. Table 1 describes both measures and defines a third measure called “within-country heterogeneity,” which we use in secondary analyses to represent the homogeneity vs. heterogeneity of values within a nation. We define all three measures here so that readers can distinguish between them in our presentation of results.
+
+
+
+
+
+ Our main analyses test whether value variation and value distinctiveness are increasing or decreasing over time. Increases indicate value divergence, and decreases indicate value convergence. We try to test this relationship as rigorously as possible. For example, we include several robustness checks to ensure that the results are not driven by the way we normalize the items or the sampling strategy of the WVS. In our main text, we complement our primary analyses by analyzing different subsets of countries and by exploring geopolitical variables that can explain country-level clustering based on value similarity. Supplemental analyses show that results replicate with different approaches to normalizing means for items that have different response scales and across demographically weighted and unweighted country-level scores. Our materials and methods summarize our analysis procedure, and the Supplementary Methods have an “extended” materials and methods section with more detail.
+
+
+ Here we use this approach to show that values have diverged across national cultures over time. This value divergence mainly characterizes a growing gap between high-income Western countries and the rest of the world on emancipative values. We also find that worldwide value divergence has been accompanied by value convergence among countries in the same region.
+
+
+
+
+
+
+
+ Results
+
+
+
+ Value divergence at the item level
+
+
+ We first examined general trends towards value convergence or divergence using our value variation measure, which allowed us to estimate effects at the item level. Our results strongly supported value divergence. A mixed effects model with value variation nested in the 40 items found that timepoint has been significantly associated with greater value variation, b = 0.004, SE = 0.0007, t(239) = 5.12, p < 0.001, \(\beta\) = 0.17, 95% CIs [0.002, 0.005]. We replicated this result using a different approach in which we correlated timepoint with value variation separately for each of the 40 value items. Of the 40 values, we found that 27 have diverged over time, with a positive median correlation of 0.28 between timepoint and value variation, t(39) = 3.30, p = 0.002, Mdiff = 0.28, 95% CIs [0.11, 0.45]. Coefficients associated with each item are displayed in Fig. 1.
+
+
+
+
+
+ Have certain kinds of values diverged more than others? To test this question, we measured how much each item related to Welzel’s dimensions of “sacred vs. secular” values and “emancipative vs. obedient” values8,27. Welzel developed these dimensions to differentiate between values that uphold or reject tradition and religion (sacred-secular values) from those that foster or restrict the freedom of the individual from the group (emancipative-obedient values)8. We found that a broad set of items loaded on the secular-sacred dimension, ranging from the justifiability of cheating on taxes (0.46), to the confidence in churches (0.46), to the justifiability of euthanasia (0.37). A narrower set of items loaded on the emancipative-obedient dimension. Of the examples above, only justifiability of euthanasia loaded above 0.35 (0.44).
+
+
+ We found that the rate of value divergence correlated with loading on the emancipative-obedient dimension, r(38) = 0.54, p < 0.001, but not the sacred-secular dimension, r(38) = 0.19, p = 0.237. This is clearly visible in Fig. 1. The 7 items with the highest divergence scores each have high loadings on the emancipative-obedient dimension. These values were (1) justifiability of homosexuality, (2) justifiability of euthanasia, (3) importance of obedience of children, (4) justifiability of divorce, (5) justifiability of prostitution, (6) justifiability of suicide, and (7) justifiability of abortion. National cultures are diverging the most on their tolerance of individual expression versus emphasis on group obedience. Supplementary Fig. 1 reproduces Fig. 1 with shading based on secular-sacred instead of emancipative-obedient loading.
+
+
+ Our approach allowed us to understand the nature and magnitude of divergence on emancipative values. From the first to the last timepoint, value variation across countries increased 141% for justifiability of homosexuality, 94% for justifiability of prostitution, 61% for justifiability of euthanasia, and 42% for importance of childhood obedience (Fig. 2).
+
+
+
+
+
+ Divergence on emancipative values is increasingly distinguishing Western countries from non-Western ones. Consider the case of Australia and Pakistan. The first time Australians were surveyed in the WVS, 39% of participants cited childhood obedience as an important quality in children, and participants rated divorce as more unjustifiable than justifiable (0.45). When Pakistanis were first surveyed, their responses were not so different: 32% cited childhood obedience as an important childhood quality and participants rated divorce as more unjustifiable than justifiable (0.10). Over time, however, these views diverged. The last time that they were surveyed, only 18% of Australians compared to 49% of Pakistanis cited childhood obedience as an important quality, and Australians viewed divorce as much more justifiable (0.74) than Pakistanis (0.15). From the 1980s to the 2020s, similar fault lines emerged between Western and non-Western countries.
+
+
+ Rises in value variation for the 7 most divergent items are displayed in Fig. 2A. The clearest rises in value variation come from timepoints 1–5 and plateau across timepoints 5–7. Figure 2B illustrates changes over time in the means of these 7 values, which are aggregated to the continent level and coded such that higher values mean more emancipative values. This plot shows that Oceanic, European, North American, and South American countries have progressively endorsed more emancipative values, whereas endorsement of these values has been stable across Asian and African countries. Divergence on some values is driven by countries moving in opposite directions, as in the case of Australia vs. Pakistan on the value of childhood obedience (see Supplementary Table 16 for other examples). Other values diverged because their mean rate of endorsement changed in some countries but remained the same in others. We provide detailed methodological details about these items and countries in our Supplementary Information (e.g., Supplementary Tables 2–5).
+
+
+ Value divergence at the country level
+
+
+ We next replicated and extended these findings using our country-level value distinctiveness measure, in which we calculated each country’s deviation from the global median of each value. We first regressed this distinctiveness score on timepoint in a mixed effects model with observations nested in values, countries, and continents. This approach helped us account for the non-independence of countries within the same continent in our data analysis. Value distinctiveness has been rising over time, b = 0.003, SE = 0.0005, t(9290) = 4.98, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.002, 0.004], indicating that countries have diverged in their values. We replicated this effect while controlling for spatial autocorrelation more continuously using an approach similar to the one we used to calculate value distinctiveness, b = 0.003, SE = 0.0005, t(9351) = 5.18, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.002, 0.004] (see Supplementary Methods for details). This continuous method further addressed the concern that our results might have been biased by interdependence of datapoints, often called “Galton’s problem.”
+
+
+ Another concern is that these findings might be confounded with cohort effects, meaning that value divergence has resulted in changes in the WVS sampling strategy over time. If the cohorts of the WVS are becoming increasingly diverse, then the mean level of value distinctiveness would rise even if countries’ values stayed consistent over time. We took several steps to address this concern. First, we used centering to separate each country’s general value distinctiveness across all waves (representing a cohort effect) from its change over time from wave to wave (representing a longitudinal effect). A model including both variables found that the longitudinal effect was significant, b = 0.003, SE = 0.0006, t(10,520) = 4.92, p < 0.001, \(\beta\) = 0.04, 95% CIs [0.002, 0.004], but the cohort effect was not, b = 0.002, SE = 0.003, t(49.82) = 0.68, p = 0.503, \(\beta\) = 0.02, 95% CIs [−0.004, 0.009].
+
+
+ Second, we recomputed value variation and replicated our models among subsamples of countries that had less turnover and hence less susceptibility to cohort changes. We replicated value divergence among the 54 countries that participated in at least 3 waves, b = 0.003, SE = 0.0006, t(8599) = 4.82, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.002, 0.004], the 32 countries that participated in at least 4 waves, b = 0.003, SE = 0.0006, t(6145) = 4.12, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.001, 0.004], and the 18 countries that participated in at least 5 waves, b = 0.003, SE = 0.0007, t(4059) = 3.73, p < 0.001, \(\beta\) = 0.05, 95% CIs [0.001, 0.004]. These results provided further evidence that value divergence is not an artifact of changing sampling strategy over time, but arose from national cultures changing in diverging directions over time.
+
+
+ Third, we replicated the finding in sample with no turnover in sample composition—a subset of 33 countries that provided data in the 1990s, 2000s, and 2010s, which were the three decades with the greatest WVS coverage. Value distinctiveness was higher in the 2000s, b = 0.01, SE = 0.003, t(78) = 4.05, p < 0.001, \(\beta\) = 0.29, 95% CIs [0.007, 0.02], and the 2010s, b = 0.01, SE = 0.003, t(78) = 2.66, p = 0.010, \(\beta\) = 0.19, 95% CIs [0.002, 0.02], compared to the 1990s. However, we found no significant difference between the 2000s vs. 2010s, b = −0.004, SE = 0.003, t(39) = −1.41, p = 0.167, \(\beta\) = −0.10, 95% CIs [−0.01, 0.002]. This decade-based analysis provides evidence for value divergence even when we fix the country sample over time. It also suggests that value divergence has been non-linear. In our Supplementary Methods, we evaluate different non-linear models of value divergence. These models suggest that the pace of value divergence has gradually slowed over time, rather than halting at a particular point.
+
+
+ Analyzing value distinctiveness across countries also allowed us to estimate which countries have become most dissimilar from the rest of the world. Inglehart’s post-materialist thesis suggests that high-income countries may hold especially distinctive values, at least in the domain of morality and tolerance. However, a range of other geopolitical variables could also lead cultures to adopt distinctive social values, including wealth inequality50, distance from the equator51, globalization19, and the presence of a liberal democracy47. We accessed data on these geopolitical metrics over time, and tested whether they could predict which countries were more distinctive and which countries were less distinctive. We retained the random effects structure from our previous mixed effects models to fit these results without violating any model assumptions.
+
+
+ In our regression analysis, GDP per capita was the only variable that significantly predicted value distinctiveness (see Table 2), b = 0.08, SE = 0.01, t(600.59) = 8.05, p < 0.001, \(\beta\) = 0.18, 95% CIs [0.06, 0.10], with the positive coefficient indicating that higher-income countries have more distinctive values than lower-income countries. No other predictors reached significance (ps > 0.05). In our Supplementary Table 8, we show that other geopolitical variables are not significantly linked to value distinctiveness, even when we remove GDP per capita from the model.
+
+
+
+
+
+ Further analyses found that the association between GDP per capita and value distinctiveness varied across world region. Wealth was associated with value distinctiveness among European countries, b = 0.08, SE = 0.02, t(18.76) = 3.63, p = 0.002, \(\beta\) = 0.18, 95% CIs [0.04, 0.12]. However, in a regression model where we interacted GDP per capita with continent dummy-codes, we found that, compared to Europe, the effect of GDP per capita was significantly weaker in Asia, b = −0.10, SE = 0.04, t(16.66) = −2.84, p = 0.011, \(\beta\) = −0.23, 95% CIs [−0.16, −0.04], and Africa, b = −0.21, SE = 0.10, t(122.60) = −2.13, p = 0.036, \(\beta\) = −0.50, 95% CIs [−0.40, −0.02]. The association between GDP per capita and value distinctiveness was non-significant in both Africa, b = −0.14, SE = 0.10, t(109.20) = −1.38, p = 0.171, \(\beta\) = −0.32, 95% CIs [−0.32, 0.06], and Asia, b = −0.02, SE = 0.03, t(16.24) = −0.78, p = 0.448, \(\beta\) = −0.05, 95% CIs [−0.08, 0.03]. These continent comparison models were based on smaller subsets of countries, with 48 countries in the Europe vs. Asia comparison (simulated power = 80.20%) and 36 countries in the Europe vs. Africa comparison (simulated power = 98.00%). Readers should therefore interpret these findings with more caution than the tests that included the full sample of countries. Supplementary Table 13 summarizes the complete set of coefficients for these models. Our Supplementary Methods provide more details about the power analysis simulations.
+
+
+ Figure 3 illustrates many of the findings that we have presented in a single plot, with countries ordered along the y-axis based on their continent and on their level of value distinctiveness within continents. The x-axis illustrates each country’s value distinctiveness score at the first WVS timepoint and the final WVS timepoint, and each country’s GDP per capita is shown on the left of the y-axis. In this visualization, it is clear that most countries have diverged from the rest of the world, with more distinctive values at the final vs. the first timepoint. Second, in Western regions but not non-Western regions, divergence has been most stark for high-income countries.
+
+
+
+
+
+ These findings suggest a provocative possibility: rises in global wealth may be responsible for the worldwide divergence of values. Most countries have become wealthier over time—Western countries but also many non-Western countries like Singapore, Hong Kong, and South Korea52. This means that most countries were poorer in the earlier (vs. later) waves of the WVS. Western nations in these early waves held more emancipative values than non-Western nations, but not by a large degree. As time passed, rising wealth led Western countries to adopt more emancipative values, but it did not have the same effect for most non-Western countries. These trends led to a growing gap between high-income Western countries and the rest of the world.
+
+
+ Consider Hong Kong and Canada, where GDP per capita has followed a similar trend, but values have diverged. Both countries had a GDP per capita of approximately $25,000 in 2000, which doubled to approximately $50,000 by 202053,54. Over the same time interval, beliefs that homosexuality was justifiable rose in Canada from 0.49/1.00 to 0.74/1.00. Perceived justifiability of homosexuality also rose in Hong Kong, but only from 0.29/1.00 to 0.44/1.00. This means that the gap in means between Hong Kong and Canada increased by 50% during this period. One of the fastest-changing values in Hong Kong during this time was belief in children’s work ethic. From 2000 to 2021, the percent of Hong Kong participants who mentioned responsibility as an important childhood quality rose from 19% to 52%, whereas it fell from 53% to 47% in Canada.
+
+
+ This example shows that wealth can sometimes lead to region-specific effects on values. If rising wealth fosters emancipative values in some regions but not in others, this could explain why Western democracies have developed more distinct values over time. This would be consistent with Eisenstadt, whose “multiple modernities” thesis states that countries follow their own trajectories of modernization19. It is also consistent with Huntington’s observations that rising wealth and influence in East Asia could lead to a re-affirmation of traditional Confucian values22. Factors such as migration, political change, and decolonialization could also contribute to these trends. For example, sovereignty over Hong Kong transitioned from the United Kingdom to China in 1997, which may have affected people’s values. More research is necessary to determine the causal relationship, if any, between wealth and value change.
+
+
+ Within-country heterogeneity and value distinctiveness
+
+
+ In our next analysis, we explored whether value distinctiveness across countries was correlated with the heterogeneity of values within countries. In other words, are countries where citizens disagree on values more like the rest of the world than countries where citizens have more homogeneous values?
+
+
+ We calculated within-country heterogeneity using the same approach that we used to calculate value distinctiveness. But instead of taking the difference of country means from the global mean across values, we took the deviation of individual ratings from the country mean. Higher levels of within-country heterogeneity therefore represent a country with high levels of disagreement among its citizens. For example, high within-country heterogeneity in the United States might arise because liberal and conservative Americans hold very different values. In exploratory analyses, we analyzed whether within-country heterogeneity was also rising across time, signaling a divergence within countries. However, we did not find general trends in this measure. Within-country heterogeneity seems to be rising among some countries and falling among others (see Supplementary Methods).
+
+
+ However, we did find that within-country heterogeneity was closely linked to value distinctiveness. When we regressed value distinctiveness on within-country heterogeneity controlling for GDP per capita, Gini, globalization, political rights, and distance from equator, within-country heterogeneity robustly and negatively predicted value distinctiveness, b = −0.06, SE = 0.01, t(8,032) = −4.45, p < 0.001, \(\beta\) = −0.07, 95% CIs [−0.09, −0.03]. In other words, countries where citizens agree more on values tend to be more different from other countries. The other effects in this model remained unchanged. GDP per capita had a positive relationship and no other variables had significant associations (see Supplementary Table 12). Figure 4 visualizes the relationship between within-country heterogeneity and value distinctiveness. Our Supplementary Methods reports timepoint specific models.
+
+
+
+
+
+ These results suggest that as countries develop more homogenous values, they may increasingly splinter from viewpoints that are shared around the rest of the world. This has happened in high-income Western nations like Sweden and Norway, where there is a higher consensus around emancipative values such as justifiability of divorce and abortion. But it has also happened in poorer nations such as China and Ghana, where consensus has formed on different values. In contrast, countries with heterogeneous values such as South Africa and India may struggle with internal division, but when averaging across their subpopulations, these countries’ values may resemble those of other countries.
+
+
+ What country characteristics predict value similarity over time?
+
+
+ In our final analysis, we explored which countries hold similar values to one another. We first fit a principal components analysis (PCA) to detect underlying dimensions in how the 40 values varied across nations. Two principal components (PCs) explained 80.70% of variation (see Materials and Methods). To understand the meaning of these dimensions, we regressed each dimension against previously developed dimensions of value variation: Welzel’s secular values and emancipative values, and Inglehart’s 12-item post-materialist index. PC1 was strongly positively linked to emancipative values, b = 0.76, SE = 0.04, t(215.60) = 19.46, p < 0.001, \(\beta\) = 0.77, 95% CIs [0.67, 0.84], and to secular values, b = 0.26, SE = 0.03, t(233.90) = 8.60, p < 0.001, \(\beta\) = 0.26, 95% CIs [0.20, 0.32]. PC2 had a smaller and less robust relationship with secular values, b = −0.13, SE = 0.07, t(194.81) = −2.10, p = 0.042, \(\beta\) = −0.13, 95% CIs [−0.27, 0.009]. No effects involving the choice index were robust (see Supplementary Methods for more information about these models).
+
+
+ We used these PCs as coordinates in a two-dimensional value space in which countries with similar values are closer together than nations with different values (see Fig. 5). We next fit regression models for each timepoint to test whether geopolitical characteristics could explain which countries were closer together in value space and which countries were further apart.
+
+
+
+
+
+ These regressions found that wealth has been the strongest correlate of value similarity across nations over time. High-income countries share values with other high-income countries, and poor countries share values with poor countries. This effect is visible in Fig. 5 (middle right), where high-income countries like New Zealand, Germany, the Netherlands, and Japan are clustered together on the right side of the value space. We also note that the left side of the value space in Fig. 5 is denser than the right side, which replicates our prior analysis: high-income countries have developed more dissimilar values compared to the rest of the world, primarily because of their unique endorsement of emancipative values.
+
+
+ We also found that geographic proximity has become progressively more correlated with value similarity over time, with an effect size rising from 0.16 (Timepoint 1) to 0.35 (Timepoint 7). This is visible in Fig. 5A: high-income East Asian countries like South Korea and Singapore are closer together than high-income Western nations like New Zealand and the Netherlands. The rising predictive power of geographic proximity suggests that values are converging within regions but are diverging across regions.
+
+
+ Religion has also emerged as a robust predictor of value similarity. Countries with more similar religious profiles have more similar values, even controlling for their similarity in wealth, geographic position, and other geopolitical features. This finding conceptually replicates a recent analysis that found that co-religionists—even those living in similar countries—shared similar values55. The fact that values are segregating along geographical and religious fault-lines further supports Huntington’s thesis that the 21st century would see a rise of ancestral cleavages in values. By the final WVS timepoint, religious similarity and geographic proximity were stronger correlates of value similarity than having similar levels of inequality or political rights. Figure 5 plots out the associations of value similarity with each form of geopolitical similarity at each timepoint. Table 3 lists key coefficients from the value similarity regressions.
+
+
+
+
+
+
+
+
+
+
+ Discussion
+
+
+
+ We find that values have diverged from 1981 to 2022 across 76 countries. Value divergence appears to be strongest for values related to tolerance and openness and can be explained by the rising difference between high-income Western countries and the rest of the world. Regional value convergence has accompanied worldwide divergence, with geographically proximal countries adopting more similar values over time.
+
+
+ Our Supplementary Methods present many robustness tests that replicate our key findings with different sampling strategies, normalization procedures, and demographic weighting. They also include an additional literature review that summarizes previous research on modern cultural differences in values and other metrics of cultural distance in greater depth.
+
+
+ These robustness analyses lend more support to our conclusions, but there are still important limitations to this study. First, we can only claim that values have diverged across the 40 items in our analysis. These items include a wide range of different values. However, they are not an exhaustive list of values across countries, and our results could change if they were replicated on a different set of items. Second, our samples are not perfectly representative of populations in the countries that we have analyzed. The WVS attempts to emulate key dimensions of representativeness, but it may sometimes neglect responses from key demographics, such as indigenous populations. For these reasons, it will be important to replicate our analyses in new datasets. Few datasets have data on social values that match the WVS, but this may change as new cross-cultural surveys are launched and analyzed over time.
+
+
+ While acknowledging these limitations, we also view our findings as potentially impactful from both a theoretical and practical standpoint. Theoretically, our findings suggest that globalization and intergroup contact alone are not sufficient to produce converging social values. Our findings also suggest that the post-materialist hypothesis—that wealth breeds emancipative values and tolerance8,9—may have stronger predictive power in some regions than others. Our findings support key parts of other theories but do not completely align with any single theory of culture and values. We observe worldwide divergence of values accompanied by re-alignment of values along regional and religious fault-lines, which is consistent with Huntington’s civilizations thesis and more recent work on rises in geopolitical regionalism from macroeconomics56. We also find that this effect of wealth varies across Western and non-Western countries, consistent with Eisenstadt’s multiple modernities theory and with other studies that highlight Asia’s unique modernization trajectory57. It may be that our findings are specific to a particular period of time following decolonization and the end of the Cold War (1981–2022) and that we would have found different results at different periods of time. Only time will tell if our findings represent a general cultural trend or a historically isolated phenomenon.
+
+
+ Value divergence could also explain theoretical puzzles in the social sciences. For example, there is a popular theory that rising wealth4 and technology58 facilitate religious decline because they decrease existential insecurity and relieve the economic pressure to have children59. But this model does not explain why rising wealth has not brought religious declines in Middle Eastern countries and has even correlated with rising religiosity in some of these countries. One possibility is that these declines will happen with time, especially with generational change. In other words, countries like Saudi Arabia and Qatar may not have been high-income long enough to experience secularization. However, another possibility is that Islam has emerged as a key part of Arabic post-colonial identity. Rising wealth and influence could have led Arab countries to emphasize the religious part of their identity to distinguish themselves from the West.
+
+
+ Our findings answer some important questions about contemporary value change, but they also pose new ones. Why has rising wealth led Western, but not non-Western, nations to espouse more emancipative values? We believe that this relationship might be embedded in ecological and cultural events throughout Western history, including the advent of participatory democracy in Athens and early Rome, enlightenment and post-enlightenment philosophy, the Catholic Church’s marriage and family polices60, the French revolution, and the reformation61. These phenomena may have gradually solidified a Western identity focused on autonomy, primacy of individual rights over obligations to the in-group, and tolerance for breaking norms2,60,62. As the world has globalized and Western nations have competed for resources on the world stage, this identity may have crystalized even further. But this does not mean that wealth or globalization should have encouraged similar values in African, East Asian, or South Asian cultures. To the contrary, growing power and resources may have prompted non-Western countries to affirm their own traditional values.
+
+
+ Practically, value divergence has implications for political polarization and conflict across world countries. Russia has framed the recent war in Ukraine as a war against Western values40. Chinese politicians have spoken against countries that “forcibly promote the concept and system of Western democracy and human rights”63. Western non-governmental organizations have faced recent accusations of seeding immorality and propagating Western imperialism64, and public opinion polling has found increasingly hostile attitudes towards Western countries in the Middle East65, Asia66, and Africa67. Our findings do not shed light on the extent of the anti-Western sentiment or the exact nature of its antecedents and consequences. We do not know to what extent governments strategically propagate certain values to reinforce national identities, as Tomlinson21 would suggest. One goal of future research could be to understand the extent that political elites have encouraged value divergence among ordinary citizens.
+
+
+ Our research also underscores the limitations of studying Westerners to make claims about human psychology writ large. Cross-cultural scholars have pointed out that people from Western, Educated, Industrialized, Rich, and Democratic (WEIRD) countries have psychological traits that differ from the rest of the world. This peculiarity presents an external validity problem for studies that recruit mostly WEIRD subjects, and it also presents an intellectual problem for cultural evolutionists who hope to explain regional variation. We show that this problem has become more acute in the last forty years. WEIRD subjects have become even more peculiar, at least in their social and moral values. This shift makes it more crucial than ever that behavioral scientists develop their theories using data from globally representative samples.
+
+
+
+
+
+
+
+ Methods
+
+
+
+ Here we report key parts of our methodology and analysis plan. Our Supplementary Methods contain an extended materials and methods section with further information.
+
+
+ Ethics
+
+
+ The director of the Social and Behavioral Science IRB office at University of Chicago determined that our research does not require IRB approval, since the nature of this research—analyzing archival deidentified data—does not quality as human subjects research.
+
+
+ The World Values Survey
+
+
+ The World Values Survey (WVS) is an international research program devoted to measuring the social, political, economic, and religious values of individuals around the world using regular surveys. The WVS website contains comprehensive information about its research procedures (https://www.worldvaluessurvey.org). This includes information about translation procedures and fieldwork training. In addition to publishing data each wave, the WVS publishes a time-series file containing data from all waves. The WVS has not surveyed the same people over time in this file. Rather, each timepoint contains a demographically representative snapshot of people in a country at a particular point in time. The WVS also publishes a list of variables indicating which items are asked in different waves, and a list of countries indicating which countries are surveyed in each wave. The timeseries dataset is published in many different formats. We downloaded the Rdata format.
+
+
+ Characteristics of countries
+
+
+ We focused on the 76 countries where the WVS has collected data for at least two waves, a necessary condition since we are interested in change over time. Supplementary Table 1 summarizes the age-sex composition of each country in each wave, with dashes indicating waves where a country was not included in data collection. Samples are designed to be representative of people age 18 and older residing within private residences in each country, regardless of citizenship or language. The WVS employs probability sampling and stratified sampling to achieve these targets. They offer case weights to compensate for small deviations with respect to gender-age (self-reported), rural-urban, or educational attainment.
+
+
+ Characteristics of Items
+
+
+ The WVS contains a heterogeneous set of items that change each year. Of the 1,011 items included in the WVS timeseries file, most were only included in a minority of waves. We focused on the items that were asked in all seven waves, and specifically on items that could be construed broadly as values. In total, we selected 40 items for analysis. Our extended Supplementary Methods summarize how we selected these items and excluded others. In Supplementary Table 2, we provide the item identification number, item label, and the scale that participants used to respond to the item. Readers can access the complete item wording for each item by downloading one of the PDF questionnaires for any wave from the WVS website.
+
+
+ One of the scales in our analysis involved the qualities that respondents felt to be important for children to learn. For these items, respondents were not allowed to select more than 5 items. However, during data processing we realized that this rule was not always followed. To keep the questionnaire format consistent across countries and waves, we excluded all respondents (n = 20,380; 5% of the total sample) who had selected more than 5 important childhood qualities. This decision did not affect our results. All results replicated with and without excluding participants who had not followed the instructions.
+
+
+ Exogenous variables
+
+
+ We collected data on exogenous variables illustrating the geopolitical conditions of countries over time. We sought to match all exogenous variables as closely as possible to the year of WVS data collection, so we downloaded time-varying estimates of these variables. We provide the source and access URL for each variable here. Our extended materials and methods provide more information about each variable.
+
+ We measured political rights using the “Political Rights” index published by the Freedom House. We accessed this variable using the “Global Economy” database. The access URL is https://www.theglobaleconomy.com/download-data.php. We note that the Freedom House publishes two different indices: a political rights index and a civil liberties index. Both indices are coded on a 7-point scale from 1 (Strong) to 7 (Weak). These correlated at 0.94 and showed identical results, so we focused on the political rights index in our analyses. We recoded the index so that higher values meant more political rights.
+
+
+ Religious distance
+
+
+ We accessed data on the % of the population identifying as Hindu, Christian, Muslim, and Buddhist, and measured religious distance by summing the absolute difference in % of people identifying with each religious group across pairs of countries. The access URL is https://www.theglobaleconomy.com/download-data.php.
+
+
+ Analytic strategy
+
+
+ This section summarizes how we computed value variation and value distinctiveness. Table 1 summarizes the meaning of these measures, along with within-country heterogeneity, which we describe in the main text.
+
+
+ Item normalization
+
+
+ Supplementary Table 2 shows that items were asked on different scales. Some items involved binary responses (e.g., whether people mention not wanting to be neighbors with someone from a specific group). Others were asked with Likert-type scales, such as the 1–10 scale that people used to rate whether behaviors were morally justifiable. We normalized the item scales using min-max normalization, which is a common approach in data science and machine learning68. Given a vector V = [v1, v2, … vn], we can determine min_V as the minimum value in the vector and max_V as the maximum value in the vector. We can create our normalized vector, V’ using:
+
+ In other words, each element in the new vector is the result of subtracting the minimum value of the original vector from that element, then dividing by the range of the original vector (i.e., the difference between its maximum and minimum values). This procedure results in variables with ranges of 0–1, no matter their original scales. As an alternative to min-max normalization, we also considered a median split approach. We discuss the limitations of this approach in the Supplementary Methods, and show how our main findings replicate with this approach.
+
+
+ Calculating value divergence across items
+
+
+ After the item-normalization procedure, we took the mean values of items for each country. This resulted in country-level means for each item at each timepoint. As an exploratory analysis, we fit Pearson correlations between timepoint and mean for each item, which indicates how values are changing over time across all countries. Supplementary Table 3 lists the full set of trends in global mean endorsement of items over time.
+
+
+ We assessed value divergence across items by computing the standard deviation of country means at each WVS timepoint—the measure of “value variation” that we summarize in our introduction. We then estimated the linear trend in these SD values in a mixed effects model which we summarize in the main text, and also across a set of Pearson correlations that we fit for each individual item and that we summarize in Fig. 1. In these models, positive coefficients represent value divergence, since the SD of country means is increasing over time. In Supplementary Table 4, we report value divergence coefficients for four different subsets of countries— countries which were included in at least 2 WVS waves, 3 waves, 4 waves, and 5 waves. We conducted our analyses for these different subsets of countries because we wanted to ensure that value divergence was not due to changing WVS composition over time.
+
+
+ We calculated the relevance of each item to Welzel’s emancipative and secular dimensions by taking the absolute value of the correlation between the item and each of the two dimensions to create continuous loading scores. Scores for both dimensions are published as variables in the WVS dataset. The item measuring participants’ view of “unselfishness” as an important child quality had a loading of 0.01 on the emancipative values dimension and a loading of 0.003 on the secular values dimension, suggesting it had low relevance for both dimensions. In contrast, the item measuring participants’ views on the justifiability of had a loading of 0.56 on the emancipative factor and a loading of 0.40 on the secular factor, reflecting higher relevance for both factors. The dimensions correlate at r = 0.36 with each other, suggesting that secular values tend to be more emancipative and that the two dimensions are not redundant.
+
+
+ Calculating value distinctiveness
+
+
+ We followed a multi-step process to calculate value distinctiveness for each country. First, we computed the global median score for each value at each timepoint. For example, the median score for the item “Important Child Qualities: Hard Work” was 0.55 in the third WVS wave. Next, we computed the absolute differences between each country’s mean and the global median for each item. For example, the average response to the “Hard Work” item in Albania was 0.57, yielding an absolute difference of 0.02. Finally, we aggregated across all of these absolute differences to obtain a “value distinctiveness” score for each country. This process of computing value distinctiveness Di,j can be expressed as:
+
+ Where ni,j represents the mean value j of a given country i. We used the median to compute the global value because it avoided outlier countries from having a large impact on the mean value.
+
+
+ Computing value distinctiveness across timepoints allowed us to estimate which countries have relatively unique values at any given time. In our main text, Fig. 3 summarizes each country’s value distinctiveness score in the first and last wave that it was included in the WVS. Supplementary Fig. 2 visualizes value distinctiveness for each country in each WVS wave, and Supplementary Table 5 summarizes value distinctiveness for each country in each wave.
+
+
+ In addition to estimating the value distinctiveness of individual countries at specific timepoints, we also analyzed general trends in value distinctiveness over time. Our reasoning was that, if countries are diverging in their values, the average value distinctiveness coefficient would be increasing. This would indicate that countries are “spreading out” around the global medians of values. One limitation of this analysis is that the global midpoint is not truly “global”—it is only the average of the countries sampled by the WVS at a particular point in time. If the WVS has become systematically more diverse in its sampling, then this could artificially create value divergence via a trend in sample heterogeneity. This is why we repeated all of our analyses for subsets of countries that had participated in 2, 3, 4, and 5 WVS waves. It is also why we conducted the decade-over-decade analysis in which we replicated the finding when looking across a subset of 33 countries which provided data in the 1990s, 2000s, and 2010s—the three decades with the greatest WVS coverage. Supplementary Table 6 summarizes value distinctiveness scores for each decade and for each country within the 33-country sample.
+
+
+ Clustering methodology
+
+
+ In our final analysis, we sought to project countries onto an n-dimensional value space. Greater distance in this space between a pair of countries would represent a larger difference between the values of two countries. We could also test which geopolitical variables were most strongly correlated with this pairwise distance metric.
+
+
+ The first step in this process was to determine how many dimensions would appropriately capture sufficient variance across values. Supplementary Fig. 3 illustrates the correlations between all 40 items in our dataset. There were clear covariances across sets of values, and so we reasoned that a small number of dimensions might explain considerable variation across values. To determine the optimal number of dimensions, we fit a Principal Components Analysis (PCA) on the correlation matrix of values (the same matrix displayed in Supplementary Fig. 3). In this analysis, PC1 explained 65.4% of variation, and PC2 explained 15.3% of variation. No other dimension explained more than 10% of variation, so we adopted a 2-dimension solution. Supplementary Fig. 3 shows an elbow plot of variance explained by each PC, and the item loadings. We projected countries onto a two-dimensional value space using these PC item loadings multiplied by the country’s scores on each value. This procedure generated the plots displayed in Fig. 5.
+
+
+ The regression that we describe in our main text (see Table 2) was fit to a dataframe where rows represented pairs of nations, with columns indicating each nation in the pairwise comparison. Subsequent columns indicated Euclidean distance in the two-dimensional value space, difference in GDP per capita, geographical distance between nations, etc. We fit a cross-classified mixed effects model with random effects for each nation in the pairwise comparison, and fixed effects entered simultaneously to control for covariation between geopolitical characteristics. We standardized all variables in the regression so that estimates were not influenced by the original measurement scales of the geopolitical variables. More positive values in these fixed effects suggest that countries with similar geopolitical characteristics also have more similar values. For example, a positive effect of GDP per capita would suggest that countries with very similar levels of wealth also have very similar social values. Changes over time in these coefficients would indicate that a geopolitical variable is becoming better or worse at predicting which countries had more similar values.
+
+ All data are available on the open science framework at https://doi.org/10.17605/OSF.IO/F9BZ7. Data on values can also be publicly downloaded from the World Values Survey at https://www.worldvaluessurvey.org/. Data on GDP per capita, annual GDP per capita growth, and the Gini coefficient were retrieved from the World Bank (https://data.worldbank.org/). Data on Gini coefficients was retrieved from the World Inequality Database (https://wid.world/data/). We retrieved data on globalization from the Swiss Economic Institute and data on political rights from the Freedom House. We used theGlobalEconomy.com, a dataset aggregator and supplier, to retrieve both datasets (https://www.theglobaleconomy.com/download-data.php). We also used theGlobalEconomy.com to retrieve our data on religious distance using the same link.
+
+ Jackson, J. C., Gelfand, M. & Ember, C. R. A global analysis of cultural tightness in non-industrial societies. Proc. R. Soc. B287, 20201036 (2020).
+
+ Norris, P. & Inglehart, R. Sacred and secular: Religion and politics worldwide. (Cambridge University Press, 2011).
+
+
+
+
+ Schulz, J. F., Bahrami-Rad, D., Beauchamp, J. P. & Henrich, J. The Church, intensive kinship, and global psychological variation. Science366, eaau5141 (2019).
+
+ Fukuyama, F. The end of history and the last man. (Simon and Schuster, 2006).
+
+
+
+
+ Fukuyama, F. The origins of political order: From prehuman times to the French Revolution. (Farrar, Straus and Giroux, 2011).
+
+
+
+
+ Giddens, A. Time, space and regionalisation. In Social relations and spatial structures. (eds. Gregory, D. & Urry, J.) 265–295 (St. Martin’s, New York, 1985).
+
+
+
+
+ Naipaul, V. S. Beyond belief: Islamic excursions among the converted peoples. (Pan Macmillan, 2012).
+
+
+
+
+ Kerr, C., Dunlop, J. T., Harbison, F. H. & Myers, C. A. Postscript to Industrialism and industrial man. Int. Lab Rev.103, 519–540 (1971).
+
+ Eisenstadt, S. N. Multiple modernities in an age of globalization. Grenzenlose Ges. Verhandlungen 29 Kongresses Dtsch. Ges. Für Soziol. 16 Kongresses Österr. Ges. Für Soziol. 11 Kongresses Schweiz. Ges. Für Soziol. Freibg. Br 1998 37–50 (1999).
+
+
+
+
+ Eisenstadt, S. N., Riedel, J. & Sachsenmaier, D. The context of the multiple modernities paradigm. in Reflections on multiple modernities 1–23 (Brill, 2002).
+
+
+
+
+ Tomlinson, J. Globalization and culture. (University of Chicago Press, 1999).
+
+
+
+
+ Huntington, S. P. The clash of civilizations? (Springer, 2000).
+
+
+
+
+ Thomson, R. et al. Relational mobility predicts social behaviors in 39 countries and is tied to historical farming and threat. Proc. Natl Acad. Sci.115, 7521–7526 (2018).
+
+ Grossmann, I. & Varnum, M. E. Social structure, infectious diseases, disasters, secularism, and cultural change in America. Psychol. Sci.26, 311–324 (2015).
+
+ Richerson, P. J. & Boyd, R. Not by genes alone: How culture transformed human evolution. (University of Chicago Press, 2008).
+
+
+
+
+ Barkow, J. H., Cosmides, L. & Tooby, J. The Adapted Mind: Evolutionary Psychology and the Generation of Culture. (Oxford University Press, 1992).
+
+
+
+
+ Sng, O., Neuberg, S. L., Varnum, M. E. & Kenrick, D. T. The behavioral ecology of cultural psychological variation. Psychol. Rev.125, 714–743 (2018).
+
+ White, C. & Muthukrishna, M. More educated, more WEIRD: Higher education predicts worldwide cultural similarity to the United States and other WEIRD countries. https://doi.org/10.31234/osf.io/8wr5d (2023).
+
+ Andersen, R. & Curtis, J. Social class, economic inequality, and the convergence of policy preferences: Evidence from 24 modern democracies. Can. Rev. Sociol. Can. Sociol.52, 266–288 (2015).
+
+ Roberts, L. L. Changing worldwide attitudes toward homosexuality: The influence of global and region-specific cultures, 1981–2012. Soc. Sci. Res.80, 114–131 (2019).
+
+ Inglehart, R. C., Nash, R., Hassan, Q. N. & Schwartzbaum, J. Attitudes toward euthanasia: a longitudinal analysis of the role of economic, cultural, and health-related factors. J. Pain. Symptom. Manag.62, 559–569 (2021).
+
+ Akaliyski, P. United in diversity? The convergence of cultural values among EU member states and candidates. Eur. J. Polit. Res.58, 388–411 (2019).
+
+ Akaliyski, P. & Welzel, C. Clashing Values: Supranational Identities, Geopolitical Rivalry and Europe’s Growing Cultural Divide. J. Cross-Cult. Psychol.51, 740–762 (2020).
+
+ Van de Vliert, E. & Van Lange, P. A. Latitudinal psychology: An ecological perspective on creativity, aggression, happiness, and beyond. Perspect. Psychol. Sci.14, 860–884 (2019).
+
+ White, C. J., Muthukrishna, M. & Norenzayan, A. Cultural similarity among coreligionists within and between countries. Proc. Natl. Acad. Sci. 118, e2109650118 (2021).
+
+
+
+
+ Pomfret, R. Is regionalism an increasing feature of the world economy? World Econ.30, 923–947 (2007).
+
+ Bomhoff, E. J. & Gu, M. M.-L. East Asia Remains Different: A Comment on the Index of “Self-Expression Values,” by Inglehart and Welzel. J. Cross-Cult. Psychol.43, 373–383 (2012).
+
+ Jackson, J. C., Yam, K. C., Tang, P. M., Sibley, C. G. & Waytz, A. Exposure to automation explains religious declines. Proc. Natl Acad. Sci.120, e2304748120 (2023).
+
+ We thank Rick Shweder, Eli Finkel, Michele Gelfand, Nour Kteily, and Nava Caluori for comments on an earlier draft of this paper. We thank the World Values Survey for providing high-quality, open-access time-varying data on values around the world.
+
+
+
+
+
+
+
+ Author information
+
+
+
+ Authors and Affiliations
+
+
+
+
+ Booth School of Business, University of Chicago, Chicago, USA
+
+ Nature Communications thanks Michael Varnum and the other, anonymous, reviewer(s) for their contribution to the peer review of this work. A peer review file is available.
+
+
+
+
+
+
+
+ Additional information
+
+
+
+ Publisher’s note Springer Nature remains neutral with regard to jurisdictional claims in published maps and institutional affiliations.
+
+ Open Access This article is licensed under a Creative Commons Attribution 4.0 International License, which permits use, sharing, adaptation, distribution and reproduction in any medium or format, as long as you give appropriate credit to the original author(s) and the source, provide a link to the Creative Commons licence, and indicate if changes were made. The images or other third party material in this article are included in the article’s Creative Commons licence, unless indicated otherwise in a credit line to the material. If material is not included in the article’s Creative Commons licence and your intended use is not permitted by statutory regulation or exceeds the permitted use, you will need to obtain permission directly from the copyright holder. To view a copy of this licence, visit http://creativecommons.org/licenses/by/4.0/.
+
+ Anyone you share the following link with will be able to read this content:
+
+
+
+ Sorry, a shareable link is not currently available for this article.
+
+
+
+
+
+
+ Provided by the Springer Nature SharedIt content-sharing initiative
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Comments
+
+
+
+ By submitting a comment you agree to abide by our Terms and Community Guidelines. If you find something abusive or that does not comply with our terms or guidelines please flag it as inappropriate.
+
+
+
+
diff --git a/test/test-pages/ourworldindata/expected-metadata.json b/test/test-pages/ourworldindata/expected-metadata.json
new file mode 100644
index 00000000..2697a5c5
--- /dev/null
+++ b/test/test-pages/ourworldindata/expected-metadata.json
@@ -0,0 +1,9 @@
+{
+ "title": "Why do we need to know about progress if we are concerned about the world's largest problems?",
+ "byline": "By: Max Roser",
+ "dir": null,
+ "excerpt": "Why have we made it our mission to publish “research and data to make progress against the world’s largest problems”?",
+ "siteName": "Our World in Data",
+ "publishedTime": null,
+ "readerable": true
+}
diff --git a/test/test-pages/ourworldindata/expected.html b/test/test-pages/ourworldindata/expected.html
new file mode 100644
index 00000000..cc9cb3db
--- /dev/null
+++ b/test/test-pages/ourworldindata/expected.html
@@ -0,0 +1,180 @@
+
+
+
+
Why have we made it our mission to publish “research and data to make progress against the world’s largest problems”?
+
+
+
originally published on June 7, 2021 (data last updated in February 2024)
Why have we made it our mission to publish “research and data to make progress against the world’s largest problems”?
+
At the heart of it is a simple truth. When we look around us, it is clear that the world faces many very large problems:
+
+
Every year, 300,000 women die from pregnancy-related causes; this means that on any average day, 830 mothers die. (source)
+
The majority of the world – 60% – lives on less than $10 per day. And almost 10% live in ‘extreme poverty,’ on less than $2.15 per day. (source)
+
The world deforested 47 million hectares of forest in the last decade; that’s an area the size of Sweden. (source)
+
60 million children of primary school age are not in school. (source)
+
Almost half of the world's population lives in non-democratic regimes. (source)
+
13% of the world’s adults do not know how to read and write. (source)
+
And 3.8% of all children die before they are five years old. This means that 5 million children die every year — 10 dead children every minute. (source)
+
+
This is a list of terrifying problems. And as we don’t hear much that would tell us otherwise, it is easy to be convinced that we can’t do anything about them. Even in the extensive 24/7 news cycles, we hear little that suggests it would be possible to make progress against these problems. The same is true for our education — questions like how to end hunger, child mortality, or deforestation are rarely part of the curriculum.
+
As a consequence, it is not surprising that many have the view that it is impossible to change the world for the better. For many large problems, the majority, in fact, believe that they are getting worse.1
+
+
This, however, is not the case. We know that it is possible to make progress against these large problems because we have already done so. Thanks to the efforts of people around the world, we have achieved progress against these problems over the course of the last generation. Each of the big problems listed above was much worse in the past:
+
+
The number of women who die from pregnancy-related causes declined from 450,000 women in the year 2000 to 300,000 per year. (source)
+
The share living in extreme poverty fell from 38% to less than 10% since 1990 – the share living on more than $10 per day increased from a quarter to more than a third. (source)
+
Global deforestation declined three-fold — from 151 million hectares in the 1980s to 47 million hectares in the 2010s. (source)
+
The number of primary school-age children who are not in school declined from over 100 million in the 1990s to 60 million. (source)
+
The share of the world living in democracies increased from 35% in 1980 to 54% today. (source)
+
The share of the world’s adults who can read and write increased from 56% in 1980 to 87% today. (source)
+
The share of children who died before they were five years old declined from 9.3% in 1990 to 3.8% – the death toll fell from 12 million dead children per year to 5 million. (sources 1 & 2)
+
+
Because our hopes and efforts for building a better future are inextricably linked to our understanding of the past, it is important to study and communicate the global development up to now. Studying our world in data and understanding how we overcame challenges that seemed insurmountable at the time should give us both confidence and guidance to tackle the problems we are currently facing. Living conditions can be improved, we know this because it has been done.
+
For each of the problems we face today, we need to also address the difficult question of whether and how we can make progress in the years ahead.
+
Let’s look at one of the very worst global problems of all from that list above, the death of children. One of the leading causes of death for children is malaria. Malaria infections kill 300,000 children in one year.2
+
+
People who study malaria see several reasons that make it very likely that continued progress against this disease is possible. The factors that are holding us back in the fight against malaria are the three factors that often limit our progress:
+
+
+ More money would make more progress possible: The world has halved the death toll from malaria within the last 15 years, largely due to insecticide-treated bed nets. Sleeping under these nets protects children from the mosquitos that transmit malaria, and paying for these nets has been shown to be an effective way to reduce malaria’s death toll.
+
+
+ More people who set themselves the goal to work towards progress can make a difference: It is not just more money that can lead to more progress against malaria. Researchers recently made good progress in the effort to develop a vaccine against malaria. Further improving this technology and making it available to all those who need it could save many lives and eventually eliminate the need for millions to sleep under bed nets every night to be protected. What is needed are skilled, hardworking people who understand the problem ahead and set themselves the goal to better solutions to the problems we face.
+
+
+ More attention and the understanding that it is a solvable problem would make more progress possible: Ultimately, I don’t think we would see a lack of funding and talented people working on malaria if malaria would get the attention it deserves. 300,000 child deaths per year means 820 dead children on any average day.
+
+
+
What is true about malaria is true about many of the problems the world faces. Making progress is hard, but it is possible.
+
The progress made over the last two centuries has not ended in our lifetimes. There are possibilities to make the world a better place. It is on us to realize this.
+
The possibility of progress should matter for what we do with our lives
+
+
It is possible to lessen substantial problems that many people are suffering under, it is possible to foresee problems that are on the horizon and reduce these risks, and it is possible to achieve changes that allow the environment around us to flourish.
+
This fact – that progress is not inevitable but possible – should matter for all of us.
+
Personally, I believe that if we are in a situation in which it is possible to make progress, then we are obliged to seek progress. But even if you do not see it as an obligation, you can also see it as an opportunity, a chance to help others, and the possibility to do useful work in your life.
+
Whether we see it as an obligation or as an opportunity, we should ask ourselves what we can do with our money, our contributions to public discourse, our democratic votes, and our lives to help the world make progress.
+
How does Our World in Data’s mission respond to this?
+
+
Our World in Data publishes research and data to make progress against the world’s largest problems.
+
We made this our mission for two reasons. We want to provide the research and data that those working towards progress need – we want to serve that community.
+
And we want more people to learn about big problems and to make the decision that they use their energy and resources to contribute to progress – we want to grow that community.
+
Serve those who work towards progress by providing research and data on the world’s largest problems
+
+
There are already many thousands of people who dedicate their lives to finding solutions to the world’s very large problems.
+
We know that many of them rely on us: engineers who build safer energy systems, activists who campaign for better health care, scientists who develop new drugs, politicians who want to learn from other countries, teachers who want to inform their students about global poverty, business founders who bring new technologies to the market, and nonprofits that need to decide where to focus their efforts.
+
Our audience is not defined by who they are but by what they care about and want to do. The people we want to serve are those who work towards progress against large problems.
+
They all need data and research to understand the problems they are working on – and to decide which problems to work on in the first place. They might want to know which diseases are most prevalent in different parts of the world, how the mix of energy sources compares across countries, or how global access to clean cooking fuels or clean water has changed over time.
+
We can serve these people well because much of the information they need already exists. The problem that we have to solve here is that this existing knowledge is often neither accessible nor understandable – it is often stored in inaccessible databases, locked away behind paywalls, and buried under jargon in academic papers. Our goal is to enable those who want to solve large problems to do their best work by increasing the use of evidence and making the existing knowledge on the big problems understandable and accessible.
+
Grow the number of people who work towards progress by challenging a culture that doesn’t make it clear that progress is possible
+
+
Our second goal is to grow the number of people who decide to contribute to progress against the largest problems.
+
There are already many people who dedicate much of their time to studying big global problems and making it central to their lives to seek ways to solve them. But we believe that it is possible to grow this group further and make it a very powerful force within the global culture because most people are concerned about the world’s problems and would want to contribute to a better future.
+
The problem, however, is that being concerned about big problems is only one precondition for someone to work towards progress. The other key requirement is that a person knows that progress is possible.3
+
+
The graphic here is a representation of our argument.
+
The big orange circle represents the many people concerned about big problems. The blue circle represents the much smaller group of people who know that progress is possible. The idea is that it is those who are part of both groups who will make the decision to work towards progress. We therefore want to grow the blue circle so that one day it becomes as large as the orange one.
+
The historian Anton Howes studies the history of innovation and writes,"People innovate because they are inspired to do so […]. And when people do not innovate, it is often simply because it never occurs to them to do so. Incentives matter too, of course. But a person needs to at least have the idea of innovation — an improving mentality — before they can choose to innovate." To make progress, we need to have the idea that progress is possible.
+
+
It is crucial to grow the number of people who know that progress is possible because almost all the world’s most pressing problems are collective action problems. These are problems that can’t be solved by anyone individually and instead require that many take them seriously and join forces to work towards a solution. When we fail to find solutions to such problems, it is because we don’t get started in the first place, and this is the situation we are in with many large problems. Instead of changing the world for the better, we get stuck in what the social scientist Alice Evans calls a ‘despondency trap’ – not knowing about positive change, we mistakenly think that positive change is impossible.
+
The fact that many believe that progress is impossible should not be too surprising. The powerful forces in our culture are not offering the perspective that progress is possible or even happening. Instead, they are often suggesting that decline is inevitable.
+
The news media is neither drawing our attention to the large problems we face nor to the fact that we are making progress against some of them. The news media focuses on daily events, but neither the big persistent problems (such as those listed above) nor the progress against them find a place in news cycles.
+
Our education systems are also not making us wonder how we can make progress, we are hardly even learning about the progress we made. Few leave school knowing about even the most fundamental achievements in how humanity improved their living conditions. The poor knowledge of the basic facts on global development is evidence for it.
+
Popular culture, too, is often detrimental. Frequently, it romanticizes a past that never was, and when it speaks about tomorrow, it paints a dystopian vision of the future rather than daring to imagine the better future that is possible.
+
Sometimes even researchers or activists reinforce the belief that progress is impossible. In their effort to emphasize the severity of the problems they themselves are concerned with, it can happen that they unwittingly present it as hopeless to change the status quo. In the worst cases, this achieves the opposite of what they want, passive fatalism rather than effective engagement.
+
And lastly some intellectuals perpetuate the idea that to believe that progress is possible is a sign of being poorly informed about the true state of the world. The difficulty of some of the world’s problems does warrant some deep pessimism, but not every problem we face justifies a pessimistic outlook, and doom and gloom should never become anyone’s intellectual default position. Sometimes the optimist is the realist, and it is the unjustified pessimism itself that is standing in the way of making progress.
+
Our team’s goal is to change this. By making the research and data on the world around us accessible and understandable, we want to offer a perspective that allows us to understand the difficulty of the problems ahead and their possible solutions.
+
We at Our World in Data believe that it is possible to grow a culture of people seeking solutions to large problems. Within each of the cultural forces I’ve just listed, some people are working towards change. There are researchers, activists, intellectuals, journalists, and teachers who are as keen on finding and communicating the solutions to move forward as they are on explaining the problems ahead. There is a strong culture of progress already, and we hope to expand it further.
+
The motivation for this work is what I have summarized at the outset: if it is possible to make progress, then we are obliged to make progress. It is not acceptable that much of the world lives in poverty, that children die, that people are hungry, that we are destroying the environment when it doesn’t have to be that way.
+
We are not saying that everything is getting better
+
+
To avoid any doubt, it is worth emphasizing what we are not saying.
+
We don’t believe that everything has gotten better. Some things have gotten much worse. Since the 1940s, we’ve had nuclear weapons that can destroy our civilization; burning fossil fuels leads to air pollution that kills millions every year; and the land use for agriculture continues to drive species into extinction.
+
And it is certainly possible that we remain stuck in the status quo or that things get worse; existential risks do not nearly receive the attention they deserve (Toby Ord’s book is an excellent overview of these risks).
+
Progress is not inevitable, and how the future turns out depends on what we do today. We are not saying we will make progress, but that we can make progress. Whether the problems we face are as old as humanity or were created by ourselves within the last decades, what matters for us now is the same. We should study these problems carefully because we can reduce our use of fossil fuels, we can give up on nuclear weapons, and we can work to bring down poverty, child mortality, and hunger.
+
These two goals belong together – we should study the world’s big problems because it is possible to make progress
+
+
A common mistake in thinking about problems and progress is to believe that focusing on one of the two would require not considering the other. That presenting the evidence for progress would mean to gloss over the problems we still have, or vice versa, that presenting the evidence on global problems would make it necessary to avoid mentioning the progress we’ve made.
+
For example, in an article in the New York Times in which he cites our work, Nick Kristof writes: “So I promise to tear my hair out every other day, but let’s interrupt our gloom for a nanosecond to note what historians may eventually see as the most important trend in the world in the early 21st century: our progress toward elimination of hideous diseases, illiteracy, and the most extreme poverty.”
+
+
I think this is not the right way of looking at it. Studying progress should not mean taking a break from the awful problems we face. These two sides belong together, it’s because we know that we can make progress that it is so important to study the problems we face.
+
+
To see this, consider the alternative. If it weren’t possible to make progress, then there would be little reason to study big problems. All we could care about was ensuring that each of us personally, and perhaps a few people close to us, were well and safe. It is the unusual time we live in – the fact that we face large problems against which we can make progress – that makes it imperative to focus on the problems we face.
+
We are interested in progress because we are not living in the best of all possible worlds.
+
Explaining progress and explaining problems belong together. We need to learn about the problems because we can make progress, and we need to learn about progress because we face large problems.
+
Conclusion
+
+
The question that guides our decisions for what we report on Our World in Data is simple: What do you need to know about our world to be able to contribute positively to the world?
+
Progress means solving problems. This makes it necessary that anyone who wants to contribute to solutions needs to study both:
+
If you care about problems, you need to study progress. The progress we achieved allows us to learn how we solved problems in the past and – most fundamentally – to know that progress is possible.
+
If you want to make progress, you need to study problems. Every problem we identify is an opportunity to make progress. To make the world a better place, the first step is to understand the problems we face today.
+
Our mission follows from this understanding. Our goal with OurWorldInData.org is to give a wide overview of the big problems the world faces, show that it is possible to make progress against even very large problems and inspire people to work on these big problems to achieve the progress that is possible.
+
We want to contribute to a culture that seeks progress – a culture of people deciding to study the very large problems we face and taking the initiative to contribute to progress against them. We want to inform thoughtful people about the world’s large problems and the possibility of progress so that they can become the engineers, politicians, voters, donors, activists, founders, or researchers that will solve them.
+
+
+ Acknowledgments: I would like to thank Hannah Ritchie, Ernst van Woerden, Charlie Giattino, Matthieu Bergel, and Esteban Ortiz-Ospina for reading drafts of this text and for their very helpful comments and ideas.
+
Besides the linked article, see Gapminder’s Ignorance Survey and Ipsos’ Perils of Perception studies for survey data on people’s perception of global problems.
+
+
+
The referenced data point refers to children under the age of 15 and comes from the WHO. The Institute of Health Metrics and Evaluation’s large-scale annual study Global Burden of Disease (GBD) estimates an even higher death toll; at 400,000 child deaths due to malaria.
+
+
+
A list of relevant studies from several different fields:
Rees, J. H., & Bamberg, S. (2014). Climate protection needs societal change: Determinants of intention to participate in collective climate action. European Journal of Social Psychology, 44(5), 466–473. doi:10.1002/ejsp.2032
Our articles and data visualizations rely on work from many different people and organizations. When citing this article, please also cite the underlying data sources. This article can be cited as:
+
+
Max Roser (2021) - “Why do we need to know about progress if we are concerned about the world's largest problems?” Published online at OurWorldInData.org. Retrieved from: 'https://ourworldindata.org/problems-and-progress' [Online Resource]
+
+
BibTeX citation
+
+
@article{owid-problems-and-progress,
+ author = {Max Roser},
+ title = {Why do we need to know about progress if we are concerned about the world's largest problems?},
+ journal = {Our World in Data},
+ year = {2021},
+ note = {https://ourworldindata.org/problems-and-progress}
+}
+
+
+
+
+
Reuse this work freely
+
All visualizations, data, and code produced by Our World in Data are completely open access under the Creative Commons BY license. You have the permission to use, distribute, and reproduce these in any medium, provided the source and authors are credited.
+
The data produced by third parties and made available by Our World in Data is subject to the license terms from the original third-party authors. We will always indicate the original source of the data in our documentation, so you should always check the license of any such third-party data before use and redistribution.
\ No newline at end of file
diff --git a/test/test-pages/ourworldindata/source.html b/test/test-pages/ourworldindata/source.html
new file mode 100644
index 00000000..a12412c2
--- /dev/null
+++ b/test/test-pages/ourworldindata/source.html
@@ -0,0 +1,596 @@
+
+
+
+
+
+ Why do we need to know about progress if we are concerned about the world's largest problems? - Our World in Data
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Why have we made it our mission to publish “research and data to make progress against the world’s largest problems”?
+
+
+ At the heart of it is a simple truth. When we look around us, it is clear that the world faces many very large problems:
+
+
+
Every year, 300,000 women die from pregnancy-related causes; this means that on any average day, 830 mothers die. (source)
+
+
The majority of the world – 60% – lives on less than $10 per day. And almost 10% live in ‘extreme poverty,’ on less than $2.15 per day. (source)
+
+
The world deforested 47 million hectares of forest in the last decade; that’s an area the size of Sweden. (source)
+
+
60 million children of primary school age are not in school. (source)
+
+
Almost half of the world's population lives in non-democratic regimes. (source)
+
+
13% of the world’s adults do not know how to read and write. (source)
+
+
And 3.8% of all children die before they are five years old. This means that 5 million children die every year — 10 dead children every minute. (source)
+
+
+
+ This is a list of terrifying problems. And as we don’t hear much that would tell us otherwise, it is easy to be convinced that we can’t do anything about them. Even in the extensive 24/7 news cycles, we hear little that suggests it would be possible to make progress against these problems. The same is true for our education — questions like how to end hunger, child mortality, or deforestation are rarely part of the curriculum.
+
+
+ As a consequence, it is not surprising that many have the view that it is impossible to change the world for the better. For many large problems, the majority, in fact, believe that they are getting worse.1
+
+
+ This, however, is not the case. We know that it is possible to make progress against these large problems because we have already done so. Thanks to the efforts of people around the world, we have achieved progress against these problems over the course of the last generation. Each of the big problems listed above was much worse in the past:
+
+
+
The number of women who die from pregnancy-related causes declined from 450,000 women in the year 2000 to 300,000 per year. (source)
+
+
The share living in extreme poverty fell from 38% to less than 10% since 1990 – the share living on more than $10 per day increased from a quarter to more than a third. (source)
+
+
Global deforestation declined three-fold — from 151 million hectares in the 1980s to 47 million hectares in the 2010s. (source)
+
+
The number of primary school-age children who are not in school declined from over 100 million in the 1990s to 60 million. (source)
+
+
The share of the world living in democracies increased from 35% in 1980 to 54% today. (source)
+
+
The share of the world’s adults who can read and write increased from 56% in 1980 to 87% today. (source)
+
+
The share of children who died before they were five years old declined from 9.3% in 1990 to 3.8% – the death toll fell from 12 million dead children per year to 5 million. (sources 1 & 2)
+
+
+
+ Because our hopes and efforts for building a better future are inextricably linked to our understanding of the past, it is important to study and communicate the global development up to now. Studying our world in data and understanding how we overcame challenges that seemed insurmountable at the time should give us both confidence and guidance to tackle the problems we are currently facing. Living conditions can be improved, we know this because it has been done.
+
+
+ For each of the problems we face today, we need to also address the difficult question of whether and how we can make progress in the years ahead.
+
+
+ Let’s look at one of the very worst global problems of all from that list above, the death of children. One of the leading causes of death for children is malaria. Malaria infections kill 300,000 children in one year.2
+
+
+ People who study malaria see several reasons that make it very likely that continued progress against this disease is possible. The factors that are holding us back in the fight against malaria are the three factors that often limit our progress:
+
+
+
+ More money would make more progress possible: The world has halved the death toll from malaria within the last 15 years, largely due to insecticide-treated bed nets. Sleeping under these nets protects children from the mosquitos that transmit malaria, and paying for these nets has been shown to be an effective way to reduce malaria’s death toll.
+
+
+ More people who set themselves the goal to work towards progress can make a difference: It is not just more money that can lead to more progress against malaria. Researchers recently made good progress in the effort to develop a vaccine against malaria. Further improving this technology and making it available to all those who need it could save many lives and eventually eliminate the need for millions to sleep under bed nets every night to be protected. What is needed are skilled, hardworking people who understand the problem ahead and set themselves the goal to better solutions to the problems we face.
+
+
+ More attention and the understanding that it is a solvable problem would make more progress possible: Ultimately, I don’t think we would see a lack of funding and talented people working on malaria if malaria would get the attention it deserves. 300,000 child deaths per year means 820 dead children on any average day.
+
+
+
+ What is true about malaria is true about many of the problems the world faces. Making progress is hard, but it is possible.
+
+
+ The progress made over the last two centuries has not ended in our lifetimes. There are possibilities to make the world a better place. It is on us to realize this.
+
+
+ The possibility of progress should matter for what we do with our lives
+
+
+ It is possible to lessen substantial problems that many people are suffering under, it is possible to foresee problems that are on the horizon and reduce these risks, and it is possible to achieve changes that allow the environment around us to flourish.
+
+
+ This fact – that progress is not inevitable but possible – should matter for all of us.
+
+
+ Personally, I believe that if we are in a situation in which it is possible to make progress, then we are obliged to seek progress. But even if you do not see it as an obligation, you can also see it as an opportunity, a chance to help others, and the possibility to do useful work in your life.
+
+
+ Whether we see it as an obligation or as an opportunity, we should ask ourselves what we can do with our money, our contributions to public discourse, our democratic votes, and our lives to help the world make progress.
+
+
+ How does Our World in Data’s mission respond to this?
+
+
+ Our World in Data publishes research and data to make progress against the world’s largest problems.
+
+
+ We made this our mission for two reasons. We want to provide the research and data that those working towards progress need – we want to serve that community.
+
+
+ And we want more people to learn about big problems and to make the decision that they use their energy and resources to contribute to progress – we want to grow that community.
+
+
+ Serve those who work towards progress by providing research and data on the world’s largest problems
+
+
+ There are already many thousands of people who dedicate their lives to finding solutions to the world’s very large problems.
+
+
+ We know that many of them rely on us: engineers who build safer energy systems, activists who campaign for better health care, scientists who develop new drugs, politicians who want to learn from other countries, teachers who want to inform their students about global poverty, business founders who bring new technologies to the market, and nonprofits that need to decide where to focus their efforts.
+
+
+ Our audience is not defined by who they are but by what they care about and want to do. The people we want to serve are those who work towards progress against large problems.
+
+
+ They all need data and research to understand the problems they are working on – and to decide which problems to work on in the first place. They might want to know which diseases are most prevalent in different parts of the world, how the mix of energy sources compares across countries, or how global access to clean cooking fuels or clean water has changed over time.
+
+
+ We can serve these people well because much of the information they need already exists. The problem that we have to solve here is that this existing knowledge is often neither accessible nor understandable – it is often stored in inaccessible databases, locked away behind paywalls, and buried under jargon in academic papers. Our goal is to enable those who want to solve large problems to do their best work by increasing the use of evidence and making the existing knowledge on the big problems understandable and accessible.
+
+
+ Grow the number of people who work towards progress by challenging a culture that doesn’t make it clear that progress is possible
+
+
+ Our second goal is to grow the number of people who decide to contribute to progress against the largest problems.
+
+
+ There are already many people who dedicate much of their time to studying big global problems and making it central to their lives to seek ways to solve them. But we believe that it is possible to grow this group further and make it a very powerful force within the global culture because most people are concerned about the world’s problems and would want to contribute to a better future.
+
+
+ The problem, however, is that being concerned about big problems is only one precondition for someone to work towards progress. The other key requirement is that a person knows that progress is possible.3
+
+
+ The graphic here is a representation of our argument.
+
+
+ The big orange circle represents the many people concerned about big problems. The blue circle represents the much smaller group of people who know that progress is possible. The idea is that it is those who are part of both groups who will make the decision to work towards progress. We therefore want to grow the blue circle so that one day it becomes as large as the orange one.
+
+
+ The historian Anton Howes studies the history of innovation and writes,"People innovate because they are inspired to do so […]. And when people do not innovate, it is often simply because it never occurs to them to do so. Incentives matter too, of course. But a person needs to at least have the idea of innovation — an improving mentality — before they can choose to innovate." To make progress, we need to have the idea that progress is possible.
+
+
+
+ It is crucial to grow the number of people who know that progress is possible because almost all the world’s most pressing problems are collective action problems. These are problems that can’t be solved by anyone individually and instead require that many take them seriously and join forces to work towards a solution. When we fail to find solutions to such problems, it is because we don’t get started in the first place, and this is the situation we are in with many large problems. Instead of changing the world for the better, we get stuck in what the social scientist Alice Evans calls a ‘despondency trap’ – not knowing about positive change, we mistakenly think that positive change is impossible.
+
+
+ The fact that many believe that progress is impossible should not be too surprising. The powerful forces in our culture are not offering the perspective that progress is possible or even happening. Instead, they are often suggesting that decline is inevitable.
+
+
+ The news media is neither drawing our attention to the large problems we face nor to the fact that we are making progress against some of them. The news media focuses on daily events, but neither the big persistent problems (such as those listed above) nor the progress against them find a place in news cycles.
+
+
+ Our education systems are also not making us wonder how we can make progress, we are hardly even learning about the progress we made. Few leave school knowing about even the most fundamental achievements in how humanity improved their living conditions. The poor knowledge of the basic facts on global development is evidence for it.
+
+
+ Popular culture, too, is often detrimental. Frequently, it romanticizes a past that never was, and when it speaks about tomorrow, it paints a dystopian vision of the future rather than daring to imagine the better future that is possible.
+
+
+ Sometimes even researchers or activists reinforce the belief that progress is impossible. In their effort to emphasize the severity of the problems they themselves are concerned with, it can happen that they unwittingly present it as hopeless to change the status quo. In the worst cases, this achieves the opposite of what they want, passive fatalism rather than effective engagement.
+
+
+ And lastly some intellectuals perpetuate the idea that to believe that progress is possible is a sign of being poorly informed about the true state of the world. The difficulty of some of the world’s problems does warrant some deep pessimism, but not every problem we face justifies a pessimistic outlook, and doom and gloom should never become anyone’s intellectual default position. Sometimes the optimist is the realist, and it is the unjustified pessimism itself that is standing in the way of making progress.
+
+
+ Our team’s goal is to change this. By making the research and data on the world around us accessible and understandable, we want to offer a perspective that allows us to understand the difficulty of the problems ahead and their possible solutions.
+
+
+ We at Our World in Data believe that it is possible to grow a culture of people seeking solutions to large problems. Within each of the cultural forces I’ve just listed, some people are working towards change. There are researchers, activists, intellectuals, journalists, and teachers who are as keen on finding and communicating the solutions to move forward as they are on explaining the problems ahead. There is a strong culture of progress already, and we hope to expand it further.
+
+
+ The motivation for this work is what I have summarized at the outset: if it is possible to make progress, then we are obliged to make progress. It is not acceptable that much of the world lives in poverty, that children die, that people are hungry, that we are destroying the environment when it doesn’t have to be that way.
+
+
+ We are not saying that everything is getting better
+
+
+ To avoid any doubt, it is worth emphasizing what we are not saying.
+
+
+ We don’t believe that everything has gotten better. Some things have gotten much worse. Since the 1940s, we’ve had nuclear weapons that can destroy our civilization; burning fossil fuels leads to air pollution that kills millions every year; and the land use for agriculture continues to drive species into extinction.
+
+
+ And it is certainly possible that we remain stuck in the status quo or that things get worse; existential risks do not nearly receive the attention they deserve (Toby Ord’s book is an excellent overview of these risks).
+
+
+ Progress is not inevitable, and how the future turns out depends on what we do today. We are not saying we will make progress, but that we can make progress. Whether the problems we face are as old as humanity or were created by ourselves within the last decades, what matters for us now is the same. We should study these problems carefully because we can reduce our use of fossil fuels, we can give up on nuclear weapons, and we can work to bring down poverty, child mortality, and hunger.
+
+
+ These two goals belong together – we should study the world’s big problems because it is possible to make progress
+
+
+ A common mistake in thinking about problems and progress is to believe that focusing on one of the two would require not considering the other. That presenting the evidence for progress would mean to gloss over the problems we still have, or vice versa, that presenting the evidence on global problems would make it necessary to avoid mentioning the progress we’ve made.
+
+
+ For example, in an article in the New York Times in which he cites our work, Nick Kristof writes: “So I promise to tear my hair out every other day, but let’s interrupt our gloom for a nanosecond to note what historians may eventually see as the most important trend in the world in the early 21st century: our progress toward elimination of hideous diseases, illiteracy, and the most extreme poverty.”
+
+
+ I think this is not the right way of looking at it. Studying progress should not mean taking a break from the awful problems we face. These two sides belong together, it’s because we know that we can make progress that it is so important to study the problems we face.
+
+
+ To see this, consider the alternative. If it weren’t possible to make progress, then there would be little reason to study big problems. All we could care about was ensuring that each of us personally, and perhaps a few people close to us, were well and safe. It is the unusual time we live in – the fact that we face large problems against which we can make progress – that makes it imperative to focus on the problems we face.
+
+
+ We are interested in progress because we are not living in the best of all possible worlds.
+
+
+ Explaining progress and explaining problems belong together. We need to learn about the problems because we can make progress, and we need to learn about progress because we face large problems.
+
+
+ Conclusion
+
+
+ The question that guides our decisions for what we report on Our World in Data is simple: What do you need to know about our world to be able to contribute positively to the world?
+
+
+ Progress means solving problems. This makes it necessary that anyone who wants to contribute to solutions needs to study both:
+
+
+ If you care about problems, you need to study progress. The progress we achieved allows us to learn how we solved problems in the past and – most fundamentally – to know that progress is possible.
+
+
+ If you want to make progress, you need to study problems. Every problem we identify is an opportunity to make progress. To make the world a better place, the first step is to understand the problems we face today.
+
+
+ Our mission follows from this understanding. Our goal with OurWorldInData.org is to give a wide overview of the big problems the world faces, show that it is possible to make progress against even very large problems and inspire people to work on these big problems to achieve the progress that is possible.
+
+
+ We want to contribute to a culture that seeks progress – a culture of people deciding to study the very large problems we face and taking the initiative to contribute to progress against them. We want to inform thoughtful people about the world’s large problems and the possibility of progress so that they can become the engineers, politicians, voters, donors, activists, founders, or researchers that will solve them.
+
+
+
+
+ Acknowledgments: I would like to thank Hannah Ritchie, Ernst van Woerden, Charlie Giattino, Matthieu Bergel, and Esteban Ortiz-Ospina for reading drafts of this text and for their very helpful comments and ideas.
+
+ Besides the linked article, see Gapminder’s Ignorance Survey and Ipsos’ Perils of Perception studies for survey data on people’s perception of global problems.
+
+
+
+
+ The referenced data point refers to children under the age of 15 and comes from the WHO. The Institute of Health Metrics and Evaluation’s large-scale annual study Global Burden of Disease (GBD) estimates an even higher death toll; at 400,000 child deaths due to malaria.
+
+
+
+
+ A list of relevant studies from several different fields:
+
+ Rees, J. H., & Bamberg, S. (2014). Climate protection needs societal change: Determinants of intention to participate in collective climate action. European Journal of Social Psychology, 44(5), 466–473. doi:10.1002/ejsp.2032
+
+ Our articles and data visualizations rely on work from many different people and organizations. When citing this article, please also cite the underlying data sources. This article can be cited as:
+
+
+
+
Max Roser (2021) - “Why do we need to know about progress if we are concerned about the world's largest problems?” Published online at OurWorldInData.org. Retrieved from: 'https://ourworldindata.org/problems-and-progress' [Online Resource]
+
+
+
+ BibTeX citation
+
+
+
+
@article{owid-problems-and-progress,
+ author = {Max Roser},
+ title = {Why do we need to know about progress if we are concerned about the world's largest problems?},
+ journal = {Our World in Data},
+ year = {2021},
+ note = {https://ourworldindata.org/problems-and-progress}
+}
+
+
+
+
+
+
+
+
+ Reuse this work freely
+
+
+ All visualizations, data, and code produced by Our World in Data are completely open access under the Creative Commons BY license. You have the permission to use, distribute, and reproduce these in any medium, provided the source and authors are credited.
+
+
+ The data produced by third parties and made available by Our World in Data is subject to the license terms from the original third-party authors. We will always indicate the original source of the data in our documentation, so you should always check the license of any such third-party data before use and redistribution.
+
+
+
+
+
+
From 94ac3c524b30c0560ac545df2b3d7dd979061bfa Mon Sep 17 00:00:00 2001
From: Shaun Martin
Date: Tue, 11 Jun 2024 11:46:50 -0500
Subject: [PATCH 2/4] fix linter issues
---
.eslintrc.js | 216 ++++++++++++++++++
Readability.js | 82 ++++---
test/generate-testcase.js | 12 +-
test/test-pages/nature/expected-metadata.json | 5 +-
test/test-pages/nature/expected.html | 3 -
.../ourworldindata/expected-metadata.json | 2 +-
test/test-readability.js | 2 +-
7 files changed, 283 insertions(+), 39 deletions(-)
create mode 100644 .eslintrc.js
diff --git a/.eslintrc.js b/.eslintrc.js
new file mode 100644
index 00000000..8f900689
--- /dev/null
+++ b/.eslintrc.js
@@ -0,0 +1,216 @@
+/* eslint-env node */
+"use strict";
+
+module.exports = {
+ "parserOptions": {
+ "ecmaVersion": 2017,
+ },
+ "env": {
+ "es6": true,
+ "shared-node-browser": true,
+ },
+ "rules": {
+ // Braces only needed for multi-line arrow function blocks
+ // "arrow-body-style": [2, "as-needed"],
+
+ // Require spacing around =>
+ // "arrow-spacing": 2,
+
+ // Always require spacing around a single line block
+ // "block-spacing": 1,
+
+ // No newline before open brace for a block
+ "brace-style": 2,
+
+ // No space before always a space after a comma
+ "comma-spacing": [2, {"before": false, "after": true}],
+
+ // Commas at the end of the line not the start
+ // "comma-style": 2,
+
+ // Don't require spaces around computed properties
+ // "computed-property-spacing": [2, "never"],
+
+ // Functions must always return something or nothing
+ "consistent-return": 2,
+
+ // Require braces around blocks that start a new line
+ // Note that this rule is likely to be overridden on a per-directory basis
+ // very frequently.
+ // "curly": [2, "multi-line"],
+
+ // Always require a trailing EOL
+ "eol-last": 2,
+
+ // Require function* name()
+ // "generator-star-spacing": [2, {"before": false, "after": true}],
+
+ // Two space indent
+ "indent": [2, 2, { "SwitchCase": 1 }],
+
+ // Space after colon not before in property declarations
+ "key-spacing": [2, { "beforeColon": false, "afterColon": true, "mode": "minimum" }],
+
+ // Unix linebreaks
+ "linebreak-style": [2, "unix"],
+
+ // Always require parenthesis for new calls
+ "new-parens": 2,
+
+ // Use [] instead of Array()
+ // "no-array-constructor": 2,
+
+ // No duplicate arguments in function declarations
+ "no-dupe-args": 2,
+
+ // No duplicate keys in object declarations
+ "no-dupe-keys": 2,
+
+ // No duplicate cases in switch statements
+ "no-duplicate-case": 2,
+
+ // No labels
+ "no-labels": 2,
+
+ // If an if block ends with a return no need for an else block
+ "no-else-return": 2,
+
+ // No empty statements
+ "no-empty": 2,
+
+ // No empty character classes in regex
+ "no-empty-character-class": 2,
+
+ // Disallow empty destructuring
+ "no-empty-pattern": 2,
+
+ // No assiging to exception variable
+ // "no-ex-assign": 2,
+
+ // No using !! where casting to boolean is already happening
+ // "no-extra-boolean-cast": 2,
+
+ // No double semicolon
+ "no-extra-semi": 2,
+
+ // No overwriting defined functions
+ "no-func-assign": 2,
+
+ // Declarations in Program or Function Body
+ "no-inner-declarations": 2,
+
+ // No invalid regular expresions
+ "no-invalid-regexp": 2,
+
+ // No odd whitespace characters
+ "no-irregular-whitespace": 2,
+
+ // No single if block inside an else block
+ "no-lonely-if": 2,
+
+ // No mixing spaces and tabs in indent
+ "no-mixed-spaces-and-tabs": [2, "smart-tabs"],
+
+ // No unnecessary spacing
+ "no-multi-spaces": [2, { exceptions: { "AssignmentExpression": true, "VariableDeclarator": true, "ArrayExpression": true, "ObjectExpression": true } }],
+
+ // No reassigning native JS objects
+ "no-native-reassign": 2,
+
+ // No (!foo in bar)
+ "no-negated-in-lhs": 2,
+
+ // Nested ternary statements are confusing
+ "no-nested-ternary": 2,
+
+ // Use {} instead of new Object()
+ // "no-new-object": 2,
+
+ // No Math() or JSON()
+ "no-obj-calls": 2,
+
+ // No octal literals
+ "no-octal": 2,
+
+ // No redeclaring variables
+ "no-redeclare": 2,
+
+ // No unnecessary comparisons
+ "no-self-compare": 2,
+
+ // No declaring variables from an outer scope
+ "no-shadow": 2,
+
+ // No declaring variables that hide things like arguments
+ "no-shadow-restricted-names": 2,
+
+ // No spaces between function name and parentheses
+ "no-spaced-func": 2,
+
+ // No trailing whitespace
+ "no-trailing-spaces": 2,
+
+ // No using undeclared variables
+ "no-undef": 2,
+
+ // Error on newline where a semicolon is needed
+ "no-unexpected-multiline": 2,
+
+ // No unreachable statements
+ "no-unreachable": 2,
+
+ // No expressions where a statement is expected
+ // "no-unused-expressions": 2,
+
+ // No declaring variables that are never used
+ "no-unused-vars": [2, {"vars": "all", "args": "none"}],
+
+ // No using variables before defined
+ // "no-use-before-define": [2, "nofunc"],
+
+ // No using with
+ "no-with": 2,
+
+ // No if/while/for blocks on the same line as the if/while/for statement:
+ "nonblock-statement-body-position": [2, "below"],
+
+ // Always require semicolon at end of statement
+ "semi": [2, "always"],
+
+ // Require space after keywords
+ "keyword-spacing": 2,
+
+ // Always use double quotes
+ "quotes": [2, "double", {"avoidEscape": true}],
+
+ // Require space before blocks
+ "space-before-blocks": 2,
+
+ // Never use spaces before function parentheses
+ // "space-before-function-paren": [2, { "anonymous": "always", "named": "never" }],
+
+ // Require spaces before finally, catch, etc.
+ // "space-before-keywords": [2, "always"],
+
+ // No space padding in parentheses
+ // "space-in-parens": [2, "never"],
+
+ // Require spaces around operators
+ // "space-infix-ops": 2,
+
+ // Require spaces after return, throw and case
+ // "space-return-throw-case": 2,
+
+ // ++ and -- should not need spacing
+ // "space-unary-ops": [2, { "words": true, "nonwords": false }],
+
+ // No comparisons to NaN
+ "use-isnan": 2,
+
+ // Only check typeof against valid results
+ "valid-typeof": 2,
+
+ // enforce trailing commas
+ "comma-dangle": ["error", "always-multiline"],
+ },
+};
diff --git a/Readability.js b/Readability.js
index 958e9973..e24ef957 100644
--- a/Readability.js
+++ b/Readability.js
@@ -1645,8 +1645,10 @@ Readability.prototype = {
}
if (!parsed["@type"] && Array.isArray(parsed["@graph"])) {
- parsed = parsed["@graph"].find(it => {
- return (it["@type"] || "").match(this.REGEXPS.jsonLdArticleTypes);
+ parsed = parsed["@graph"].find(function(it) {
+ return (it["@type"] || "").match(
+ this.REGEXPS.jsonLdArticleTypes,
+ );
});
}
@@ -1719,6 +1721,36 @@ Readability.prototype = {
return metadata ? metadata : {};
},
+ /**
+ * Swaps the "Surname, GivenName" formatted bylines to "GivenName Surname".
+ *
+ * @param {string|string[]} name
+ * @returns Name or names in "GivenName Surname" format
+ */
+ _normalizeByline: function(name) {
+ var result = name;
+
+ if (Array.isArray(name)) {
+ return name.map((n) => this._normalizeByline(n));
+ }
+
+ // handle Surname, GivenName formatting
+ if (name.includes(",")) {
+ const parts = name.split(",").map(part => part.trim());
+ if (parts.length == 2) {
+ result = `${parts[1]} ${parts[0]}`;
+ }
+ if (parts.length > 2) {
+ result = `${parts[1]} ${parts[0]} ${parts.slice(2).join(" ")}`;
+ }
+ }
+
+ // remove things like "By:"
+ result = result.replace(/\w+:/, "");
+
+ return this._unescapeHtmlEntities(result);
+ },
+
/**
* Attempts to get excerpt and byline metadata for the article.
*
@@ -1737,12 +1769,7 @@ Readability.prototype = {
/\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;
// name is a single value
-<<<<<<< HEAD
- var namePattern =
- /^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;
-=======
var namePattern = /^\s*(?:(prism|citation|dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-_\.:]\s*)?(author|creator|pub-date|publicationDate|publication|description|title|site_name)\s*$/i;
->>>>>>> 740ddd3 (WIP: add citation, prism, and dc metadata)
// Find description tags.
this._forEachNode(metaElements, function (element) {
@@ -1754,6 +1781,7 @@ Readability.prototype = {
}
var matches = null;
var name = null;
+ var result = null;
if (elementProperty) {
matches = elementProperty.match(propertyPattern);
@@ -1762,7 +1790,7 @@ Readability.prototype = {
// so we can match below.
name = matches[0].toLowerCase().replace(/\s/g, "");
// multiple authors
- values[name] = content.trim();
+ result = content.trim();
}
}
if (!matches && elementName && namePattern.test(elementName)) {
@@ -1771,8 +1799,23 @@ Readability.prototype = {
// Convert to lowercase, remove any whitespace, and convert dots
// to colons so we can match below.
name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
- values[name] = content.trim();
+ result = content.trim();
+ }
+ }
+
+ // handle properties which might have multiple distinct values, eg: citation_author
+ if (result) {
+ if (values[name]) {
+ if (Array.isArray(values[name]) && typeof result == "string") {
+ values[name].push(result);
+ }
+ if (typeof values[name] == "string") {
+ values[name] = [values[name], result];
+ }
+ } else {
+ values[name] = result;
}
+ this.log(`found metadata: ${name}=${values[name]}`);
}
});
@@ -1793,21 +1836,12 @@ Readability.prototype = {
}
// get author
-<<<<<<< HEAD
- metadata.byline =
- jsonld.byline ||
- values["dc:creator"] ||
- values["dcterm:creator"] ||
- values.author ||
- values["parsely-author"];
-=======
metadata.byline = jsonld.byline ||
values["dc:creator"] ||
values["dcterm:creator"] ||
- values["author"] ||
+ values.author ||
values["parsely-author"] ||
values["citation_author"];
->>>>>>> 740ddd3 (WIP: add citation, prism, and dc metadata)
// get description
metadata.excerpt =
@@ -1824,25 +1858,17 @@ Readability.prototype = {
metadata.siteName = jsonld.siteName || values["og:site_name"];
// get article published time
-<<<<<<< HEAD
- metadata.publishedTime =
- jsonld.datePublished ||
- values["article:published_time"] ||
- values["parsely-pub-date"] ||
- null;
-=======
metadata.publishedTime = jsonld.datePublished ||
values["article:published_time"] ||
values["parsely-pub-date"] ||
values["citation_publication_date"] ||
values["prism:publicationDate"] ||
null;
->>>>>>> 740ddd3 (WIP: add citation, prism, and dc metadata)
// in many sites the meta value is escaped with HTML entities,
// so here we need to unescape it
metadata.title = this._unescapeHtmlEntities(metadata.title);
- metadata.byline = this._unescapeHtmlEntities(metadata.byline);
+ metadata.byline = this._normalizeByline(metadata.byline);
metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);
diff --git a/test/generate-testcase.js b/test/generate-testcase.js
index 82d3a477..6c2079c9 100644
--- a/test/generate-testcase.js
+++ b/test/generate-testcase.js
@@ -67,8 +67,9 @@ function getWithRedirects(url, cb) {
console.log("HEADERS:", JSON.stringify(response.headers));
}
- if(response.statusCode > 300 && response.statusCode <= 303) {
- if (debug) console.log("following redirect", response.headers.location);
+ if (response.statusCode > 300 && response.statusCode <= 303) {
+ if (debug)
+ console.log("following redirect", response.headers.location);
await getWithRedirects(response.headers.location, cb);
}
@@ -78,9 +79,10 @@ function getWithRedirects(url, cb) {
rv += chunk;
});
- response.on("end", function () => {
- if (debug) console.log("End received");
- sanitizeSource(rv, cb);
+ response.on("end", () => {
+ if (debug)
+ console.log("End received");
+ cb(rv);
});
});
}
diff --git a/test/test-pages/nature/expected-metadata.json b/test/test-pages/nature/expected-metadata.json
index d87c2608..e7e5a4ae 100644
--- a/test/test-pages/nature/expected-metadata.json
+++ b/test/test-pages/nature/expected-metadata.json
@@ -1,6 +1,9 @@
{
"title": "Worldwide divergence of values",
- "byline": "Medvedev, Danila",
+ "byline": [
+ "Joshua Conrad Jackson",
+ "Danila Medvedev"
+ ],
"dir": null,
"lang": "en",
"excerpt": "Social scientists have long debated the nature of cultural change in a modernizing and globalizing world. Some scholars predicted that national cultures would converge by adopting social values typical of Western democracies. Others predicted that cultural differences in values would persist or even increase over time. We test these competing predictions by analyzing survey data from 1981 to 2022 (n = 406,185) from 76 national cultures. We find evidence of global value divergence. Values emphasizing tolerance and self-expression have diverged most sharply, especially between high-income Western countries and the rest of the world. We also find that countries with similar per-capita GDP levels have held similar values over the last 40 years. Over time, however, geographic proximity has emerged as an increasingly strong correlate of value similarity, indicating that values have diverged globally but converged regionally. The authors test whether social values have become converged or diverged across national cultures over the last 40 years using a 76-country analysis of the World Values Survey. They show that values have diverged, especially between high-income Western countries and the rest of the world.",
diff --git a/test/test-pages/nature/expected.html b/test/test-pages/nature/expected.html
index 4e2133a4..2b515bb2 100644
--- a/test/test-pages/nature/expected.html
+++ b/test/test-pages/nature/expected.html
@@ -651,9 +651,6 @@
Additional information
Publisher’s note Springer Nature remains neutral with regard to jurisdictional claims in published maps and institutional affiliations.
-
-
Supplementary information
-
Rights and permissions
diff --git a/test/test-pages/ourworldindata/expected-metadata.json b/test/test-pages/ourworldindata/expected-metadata.json
index 2697a5c5..6c5f60f7 100644
--- a/test/test-pages/ourworldindata/expected-metadata.json
+++ b/test/test-pages/ourworldindata/expected-metadata.json
@@ -1,6 +1,6 @@
{
"title": "Why do we need to know about progress if we are concerned about the world's largest problems?",
- "byline": "By: Max Roser",
+ "byline": "Max Roser",
"dir": null,
"excerpt": "Why have we made it our mission to publish “research and data to make progress against the world’s largest problems”?",
"siteName": "Our World in Data",
diff --git a/test/test-readability.js b/test/test-readability.js
index 8646c7f2..7dc0093b 100644
--- a/test/test-readability.js
+++ b/test/test-readability.js
@@ -339,7 +339,7 @@ describe("Readability API", function () {
it("should use custom video regex sent as option", function () {
var dom = new JSDOM(
"
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mollis leo lacus, vitae semper nisl ullamcorper ut.
" +
- ''
+ "",
);
var expected_xhtml =
'
' +
From 386317815a0b5087a4494f66d697264c1e4b4939 Mon Sep 17 00:00:00 2001
From: Shaun Martin
Date: Tue, 11 Jun 2024 15:33:27 -0500
Subject: [PATCH 3/4] fix tests
---
Readability.js | 112 +++++++++++++-----
.../expected-metadata.json | 2 +-
.../003-metadata-preferred/source.html | 8 +-
.../test-pages/ebb-org/expected-metadata.json | 2 +-
test/test-pages/ietf-1/expected-metadata.json | 5 +-
test/test-pages/nature/expected-metadata.json | 4 +-
6 files changed, 95 insertions(+), 38 deletions(-)
diff --git a/Readability.js b/Readability.js
index e24ef957..d6442447 100644
--- a/Readability.js
+++ b/Readability.js
@@ -1607,6 +1607,58 @@ Readability.prototype = {
});
},
+ _extractJSONLDMetadata: function (parsed) {
+ var metadata = {};
+
+ if (typeof parsed.name === "string" && typeof parsed.headline === "string" && parsed.name !== parsed.headline) {
+ // we have both name and headline element in the JSON-LD. They should both be the same but some websites like aktualne.cz
+ // put their own name into "name" and the article title to "headline" which confuses Readability. So we try to check if either
+ // "name" or "headline" closely matches the html title, and if so, use that one. If not, then we use "name" by default.
+
+ var title = this._getArticleTitle();
+ var nameMatches = this._textSimilarity(parsed.name, title) > 0.75;
+ var headlineMatches = this._textSimilarity(parsed.headline, title) > 0.75;
+
+ if (headlineMatches && !nameMatches) {
+ metadata.title = parsed.headline;
+ } else {
+ metadata.title = parsed.name;
+ }
+ } else if (typeof parsed.name === "string") {
+ metadata.title = parsed.name.trim();
+ } else if (typeof parsed.headline === "string") {
+ metadata.title = parsed.headline.trim();
+ }
+ if (parsed.author) {
+ if (typeof parsed.author.name === "string") {
+ metadata.byline = parsed.author.name.trim();
+ } else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === "string") {
+ metadata.byline = parsed.author
+ .filter(function(author) {
+ return author && typeof author.name === "string";
+ })
+ .map(function(author) {
+ return author.name.trim();
+ })
+ .join(", ");
+ }
+ }
+ if (typeof parsed.description === "string") {
+ metadata.excerpt = parsed.description.trim();
+ }
+ if (
+ parsed.publisher &&
+ typeof parsed.publisher.name === "string"
+ ) {
+ metadata.siteName = parsed.publisher.name.trim();
+ }
+ if (typeof parsed.datePublished === "string") {
+ metadata.datePublished = parsed.datePublished.trim();
+ }
+
+ return metadata;
+ },
+
/**
* Try to extract metadata from JSON-LD object.
* For now, only Schema.org objects of type Article or its subtypes are supported.
@@ -1630,13 +1682,6 @@ Readability.prototype = {
);
var parsed = JSON.parse(content);
- // some sites, like ones for academic journals, separate metadata for a journal article or paper from the
- // site's own metadata. eg: nature has only @context, @type (WebPage), and mainEntity so *all* relevant metadata
- // would be invisible without this.
- if (parsed["mainEntity"]) {
- parsed = parsed["mainEntity"];
- }
-
if (
!parsed["@context"] ||
!parsed["@context"].match(/^https?\:\/\/schema\.org\/?$/)
@@ -1660,7 +1705,7 @@ Readability.prototype = {
return;
}
- metadata = {};
+ metadata = this._extractJSONLDMetadata(parsed);
if (
typeof parsed.name === "string" &&
@@ -1713,6 +1758,13 @@ Readability.prototype = {
if (typeof parsed.datePublished === "string") {
metadata.datePublished = parsed.datePublished.trim();
}
+ // some sites, like ones for academic journals, separate metadata for a journal article or paper from the
+ // site's own metadata. eg: nature has only @context, @type (WebPage), and mainEntity so *all* relevant metadata
+ // would be invisible unless we retry using mainEntity.
+ if (parsed["mainEntity"] && Object.keys(metadata).length === 0) {
+ metadata = this._extractJSONLDMetadata(parsed["mainEntity"]);
+ }
+
} catch (err) {
this.log(err.message);
}
@@ -1728,25 +1780,18 @@ Readability.prototype = {
* @returns Name or names in "GivenName Surname" format
*/
_normalizeByline: function(name) {
+ if (!name) {
+ return name;
+ }
+
var result = name;
if (Array.isArray(name)) {
return name.map((n) => this._normalizeByline(n));
}
- // handle Surname, GivenName formatting
- if (name.includes(",")) {
- const parts = name.split(",").map(part => part.trim());
- if (parts.length == 2) {
- result = `${parts[1]} ${parts[0]}`;
- }
- if (parts.length > 2) {
- result = `${parts[1]} ${parts[0]} ${parts.slice(2).join(" ")}`;
- }
- }
-
- // remove things like "By:"
- result = result.replace(/\w+:/, "");
+ // remove things like "By:" and "http://"
+ result = result.replace(/\w+:\/{0,2}/, "");
return this._unescapeHtmlEntities(result);
},
@@ -1771,6 +1816,11 @@ Readability.prototype = {
// name is a single value
var namePattern = /^\s*(?:(prism|citation|dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-_\.:]\s*)?(author|creator|pub-date|publicationDate|publication|description|title|site_name)\s*$/i;
+ // fields which are permitted to have multiple distinct values, eg: byline
+ var byline_properties = [ "dc:creator", "dcterm:creator", "author", "parsely-author", "citation_author"];
+ var multi_props = byline_properties; // concat others here. somewhat pointless atm, but there will be more...
+
+
// Find description tags.
this._forEachNode(metaElements, function (element) {
var elementName = element.getAttribute("name");
@@ -1803,18 +1853,19 @@ Readability.prototype = {
}
}
- // handle properties which might have multiple distinct values, eg: citation_author
if (result) {
- if (values[name]) {
+ // handle properties which might have multiple distinct values
+ if (values[name] && multi_props.includes(name)) {
if (Array.isArray(values[name]) && typeof result == "string") {
values[name].push(result);
}
- if (typeof values[name] == "string") {
+ if (typeof values[name] == "string" && values[name] !== result) {
values[name] = [values[name], result];
}
} else {
values[name] = result;
}
+
this.log(`found metadata: ${name}=${values[name]}`);
}
});
@@ -1836,12 +1887,12 @@ Readability.prototype = {
}
// get author
- metadata.byline = jsonld.byline ||
- values["dc:creator"] ||
- values["dcterm:creator"] ||
- values.author ||
- values["parsely-author"] ||
- values["citation_author"];
+ metadata.byline = jsonld.byline;
+ for (const n of byline_properties) {
+ if (metadata.byline)
+ break;
+ metadata.byline = values[n];
+ }
// get description
metadata.excerpt =
@@ -1872,6 +1923,7 @@ Readability.prototype = {
metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);
+ this.log(`getArticleMetadata complete: ${JSON.stringify(metadata)}`);
return metadata;
},
diff --git a/test/test-pages/003-metadata-preferred/expected-metadata.json b/test/test-pages/003-metadata-preferred/expected-metadata.json
index c21bd5c3..3d20de65 100644
--- a/test/test-pages/003-metadata-preferred/expected-metadata.json
+++ b/test/test-pages/003-metadata-preferred/expected-metadata.json
@@ -1,6 +1,6 @@
{
"title": "Dublin Core property title",
- "byline": "Dublin Core property author",
+ "byline": "Dublin Core author",
"dir": null,
"excerpt": "Dublin Core property description",
"siteName": null,
diff --git a/test/test-pages/003-metadata-preferred/source.html b/test/test-pages/003-metadata-preferred/source.html
index c7275b3b..4317879a 100644
--- a/test/test-pages/003-metadata-preferred/source.html
+++ b/test/test-pages/003-metadata-preferred/source.html
@@ -11,9 +11,11 @@
-
-
-
+
+
+
+
diff --git a/test/test-pages/ebb-org/expected-metadata.json b/test/test-pages/ebb-org/expected-metadata.json
index 3226ddaa..f3c3d9dd 100644
--- a/test/test-pages/ebb-org/expected-metadata.json
+++ b/test/test-pages/ebb-org/expected-metadata.json
@@ -1,6 +1,6 @@
{
"title": "On Recent Controversial Events - Bradley M. Kuhn ( Brad ) ( bkuhn )",
- "byline": "Bradley M. Kuhn (http://ebb.org/bkuhn/)",
+ "byline": "Bradley M. Kuhn (ebb.org/bkuhn/)",
"dir": null,
"lang": "en-US",
"excerpt": "The website of Bradley M. Kuhn, aka Brad, aka bkuhn. This site includes his GPG keys, resume, blog, projects list, software, interviews, speeches and writing.",
diff --git a/test/test-pages/ietf-1/expected-metadata.json b/test/test-pages/ietf-1/expected-metadata.json
index 7fc95f5f..95fa0b04 100644
--- a/test/test-pages/ietf-1/expected-metadata.json
+++ b/test/test-pages/ietf-1/expected-metadata.json
@@ -1,6 +1,9 @@
{
"title": "remoteStorage",
- "byline": "Jong, Michiel de",
+ "byline": [
+ "Kooman, F.",
+ "Jong, Michiel de"
+ ],
"dir": null,
"lang": "en",
"siteName": null,
diff --git a/test/test-pages/nature/expected-metadata.json b/test/test-pages/nature/expected-metadata.json
index e7e5a4ae..d70f5206 100644
--- a/test/test-pages/nature/expected-metadata.json
+++ b/test/test-pages/nature/expected-metadata.json
@@ -1,8 +1,8 @@
{
"title": "Worldwide divergence of values",
"byline": [
- "Joshua Conrad Jackson",
- "Danila Medvedev"
+ "Jackson, Joshua Conrad",
+ "Medvedev, Danila"
],
"dir": null,
"lang": "en",
From eb8fc7744faaf577554183499eb6f1d8caba9956 Mon Sep 17 00:00:00 2001
From: Shaun Martin
Date: Tue, 11 Jun 2024 15:51:44 -0500
Subject: [PATCH 4/4] update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1b6870d4..188e824e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ reasonable).
- [Add Parsely tags as a fallback metadata source](https://github.com/mozilla/readability/pull/865)
- [Fix the case that jsonld parse process is ignored when context url include the trailing slash](https://github.com/mozilla/readability/pull/833)
- [Fixed situations where short paragraphs of legitimate content would be excluded](https://github.com/mozilla/readability/pull/867)
+- [Add `citation`, `prism`, and more `dc` metadata](https://github.com/mozilla/readability/pull/871)
## [0.5.0] - 2023-12-15