From d3163008e7a471364453ba9fb9cb7899f37c31a6 Mon Sep 17 00:00:00 2001 From: Jacob Wenger Date: Sun, 16 Jun 2024 20:41:32 -0700 Subject: [PATCH] Mostly there --- sdow/helpers.py | 3 +- website/src/api.ts | 4 +- website/src/components/Home.tsx | 2 +- website/src/components/Loading.tsx | 2 +- website/src/components/PageInput.tsx | 21 +- website/src/components/PageInputSuggestion.js | 18 -- ...tion.styles.js => PageInputSuggestion.tsx} | 31 +- website/src/components/Results.tsx | 4 +- website/src/components/ResultsGraph.tsx | 268 +++++++++++++----- website/src/components/ResultsList.tsx | 2 +- .../posts/SearchResultsAnalysisPost/data.js | 2 +- .../posts/SearchResultsAnalysisPost/index.js | 2 +- .../resources/{constants.js => constants.ts} | 0 website/src/types.ts | 1 + website/src/{utils.js => utils.ts} | 12 +- 15 files changed, 261 insertions(+), 111 deletions(-) delete mode 100644 website/src/components/PageInputSuggestion.js rename website/src/components/{PageInputSuggestion.styles.js => PageInputSuggestion.tsx} (50%) rename website/src/resources/{constants.js => constants.ts} (100%) rename website/src/{utils.js => utils.ts} (73%) diff --git a/sdow/helpers.py b/sdow/helpers.py index 573df66..ab0d43f 100755 --- a/sdow/helpers.py +++ b/sdow/helpers.py @@ -61,13 +61,14 @@ def fetch_wikipedia_pages_info(page_ids, database): raise ValueError('Empty MediaWiki API response') for page_id, page in pages_result.items(): - page_id = str(page_id) + page_id = int(page_id) if 'missing' in page: # If the page has been deleted since the current Wikipedia database dump, fetch the page # title from the SDOW database and create the (albeit broken) URL. page_title = database.fetch_page_title(page_id) pages_info[page_id] = { + 'id': page_id, 'title': page_title, 'url': 'https://en.wikipedia.org/wiki/{0}'.format(page_title) } diff --git a/website/src/api.ts b/website/src/api.ts index dd3e705..a23b470 100644 --- a/website/src/api.ts +++ b/website/src/api.ts @@ -1,5 +1,5 @@ -import {SDOW_API_URL} from './resources/constants'; -import {ShortestPathsApiResponse, WikipediaPage, WikipediaPageId} from './types'; +import {SDOW_API_URL} from './resources/constants.ts'; +import {ShortestPathsApiResponse, WikipediaPage, WikipediaPageId} from './types.ts'; interface FetchShortestPathsResponse { readonly paths: readonly WikipediaPageId[][]; diff --git a/website/src/components/Home.tsx b/website/src/components/Home.tsx index ad2e593..48cb389 100644 --- a/website/src/components/Home.tsx +++ b/website/src/components/Home.tsx @@ -3,7 +3,7 @@ import React, {useCallback, useState} from 'react'; import {fetchShortestPaths} from '../api.ts'; import {WikipediaPage, WikipediaPageId} from '../types.ts'; -import {getRandomPageTitle} from '../utils.js'; +import {getRandomPageTitle} from '../utils.ts'; import {Logo} from './common/Logo'; import {ErrorMessage} from './ErrorMessage.tsx'; import {InputFlexContainer, Modal, P} from './Home.styles.ts'; diff --git a/website/src/components/Loading.tsx b/website/src/components/Loading.tsx index 076574c..7ce3e0c 100644 --- a/website/src/components/Loading.tsx +++ b/website/src/components/Loading.tsx @@ -1,7 +1,7 @@ import React, {useEffect, useState} from 'react'; import styled from 'styled-components'; -import {getRandomWikipediaFact, getWikipediaPageUrl} from '../utils'; +import {getRandomWikipediaFact, getWikipediaPageUrl} from '../utils.ts'; import {StyledTextLink} from './common/StyledTextLink.tsx'; const Wrapper = styled.div` diff --git a/website/src/components/PageInput.tsx b/website/src/components/PageInput.tsx index b4a9f9a..2c3fb85 100644 --- a/website/src/components/PageInput.tsx +++ b/website/src/components/PageInput.tsx @@ -6,10 +6,10 @@ import React, {useCallback, useEffect, useState} from 'react'; import Autosuggest from 'react-autosuggest'; import styled from 'styled-components'; -import {SDOW_USER_AGENT, WIKIPEDIA_API_URL} from '../resources/constants'; -import {WikipediaPage} from '../types'; -import {getRandomPageTitle} from '../utils'; -import {PageInputSuggestion} from './PageInputSuggestion'; +import {SDOW_USER_AGENT, WIKIPEDIA_API_URL} from '../resources/constants.ts'; +import {WikipediaPage} from '../types.ts'; +import {getRandomPageTitle} from '../utils.ts'; +import {PageInputSuggestion} from './PageInputSuggestion.tsx'; type PageSuggestion = Required>; @@ -100,7 +100,13 @@ const AutosuggestWrapper = styled.div` // Autosuggest component helpers. const getSuggestionValue = (suggestion) => suggestion.title; -const renderSuggestion = (suggestion) => ; +const renderSuggestion = (suggestion) => ( + +); export const PageInput: React.FC<{ readonly title: string; @@ -152,11 +158,14 @@ export const PageInput: React.FC<{ const pageResults = get(data, 'query.pages', {}); const newSuggestions: PageSuggestion[] = []; - forEach(pageResults, ({ns, title, terms, thumbnail}) => { + forEach(pageResults, (all) => { + const {ns, id, title, terms, thumbnail} = all; + console.log('ALL:', all); if (ns === 0) { let description = get(terms, 'description.0', ''); description = description.charAt(0).toUpperCase() + description.slice(1); newSuggestions.push({ + id, title, description, thumbnailUrl: get(thumbnail, 'source'), diff --git a/website/src/components/PageInputSuggestion.js b/website/src/components/PageInputSuggestion.js deleted file mode 100644 index 8b46c5e..0000000 --- a/website/src/components/PageInputSuggestion.js +++ /dev/null @@ -1,18 +0,0 @@ -import React from 'react'; - -import defaultPageThumbnail from '../images/defaultPageThumbnail.png'; -import {Description, Image, InnerWrapper, Title, Wrapper} from './PageInputSuggestion.styles'; - -export const PageInputSuggestion = ({title, description, thumbnailUrl}) => { - const descriptionContent = description ? {description} : null; - - return ( - - - - {title} - {descriptionContent} - - - ); -}; diff --git a/website/src/components/PageInputSuggestion.styles.js b/website/src/components/PageInputSuggestion.tsx similarity index 50% rename from website/src/components/PageInputSuggestion.styles.js rename to website/src/components/PageInputSuggestion.tsx index 42585b1..2e02855 100644 --- a/website/src/components/PageInputSuggestion.styles.js +++ b/website/src/components/PageInputSuggestion.tsx @@ -1,6 +1,9 @@ +import React from 'react'; import styled from 'styled-components'; -export const Wrapper = styled.div` +import defaultPageThumbnail from '../images/defaultPageThumbnail.png'; + +const Wrapper = styled.div` width: 100%; display: flex; align-items: center; @@ -8,7 +11,7 @@ export const Wrapper = styled.div` color: ${({theme}) => theme.colors.darkGreen}; `; -export const InnerWrapper = styled.div` +const InnerWrapper = styled.div` display: flex; flex-direction: column; justify-content: center; @@ -20,7 +23,7 @@ export const InnerWrapper = styled.div` } `; -export const Image = styled.img` +const Image = styled.img` width: 60px; height: 60px; margin-right: 12px; @@ -33,7 +36,7 @@ export const Image = styled.img` } `; -export const Title = styled.p` +const Title = styled.p` font-size: 20px; @media (max-width: 600px) { @@ -41,7 +44,7 @@ export const Title = styled.p` } `; -export const Description = styled.p` +const Description = styled.p` font-size: 12px; max-height: 48px; overflow: hidden; @@ -50,3 +53,21 @@ export const Description = styled.p` max-height: 32px; } `; + +export const PageInputSuggestion: React.FC<{ + readonly title: string; + readonly description: string; + readonly thumbnailUrl?: string; +}> = ({title, description, thumbnailUrl}) => { + const descriptionContent = description ? {description} : null; + + return ( + + + + {title} + {descriptionContent} + + + ); +}; diff --git a/website/src/components/Results.tsx b/website/src/components/Results.tsx index 3a8f883..798b4f4 100644 --- a/website/src/components/Results.tsx +++ b/website/src/components/Results.tsx @@ -1,8 +1,8 @@ import React from 'react'; import styled from 'styled-components'; -import {WikipediaPage, WikipediaPageId} from '../types'; -import {getNumberWithCommas, getWikipediaPageUrl} from '../utils'; +import {WikipediaPage, WikipediaPageId} from '../types.ts'; +import {getNumberWithCommas, getWikipediaPageUrl} from '../utils.ts'; import {Button} from './common/Button.tsx'; import {StyledTextLink} from './common/StyledTextLink.tsx'; import {ResultsGraph} from './ResultsGraph.tsx'; diff --git a/website/src/components/ResultsGraph.tsx b/website/src/components/ResultsGraph.tsx index 788d4cb..ebba174 100644 --- a/website/src/components/ResultsGraph.tsx +++ b/website/src/components/ResultsGraph.tsx @@ -5,8 +5,8 @@ import range from 'lodash/range'; import React, {useCallback, useEffect, useMemo, useRef} from 'react'; import styled from 'styled-components'; -import {WikipediaPage, WikipediaPageId} from '../types'; -import {getWikipediaPageUrl} from '../utils'; +import {WikipediaPage, WikipediaPageId} from '../types.ts'; +import {getWikipediaPageUrl} from '../utils.ts'; import {Button} from './common/Button.tsx'; const DEFAULT_CHART_HEIGHT = 600; @@ -118,22 +118,57 @@ const ResetButton = styled(Button)` } `; -interface GraphNode { +interface GraphNode extends d3.SimulationNodeDatum { readonly id: WikipediaPageId; readonly title: string; readonly degree: number; } -interface GraphLink { - readonly source: number; - readonly target: number; +interface GraphLink extends d3.SimulationLinkDatum { + readonly source: GraphNode; + readonly target: GraphNode; } +const GraphLegend: React.FC<{ + readonly paths: readonly WikipediaPageId[][]; + readonly color: d3.ScaleOrdinal; +}> = ({paths, color}) => { + const labels = map(range(0, paths[0].length), (i) => { + if (i === 0 && paths[0].length === 1) { + return 'Start / end page'; + } else if (i === 0) { + return 'Start page'; + } else if (i === paths[0].length - 1) { + return 'End page'; + } else { + const degreeOrDegrees = i === 1 ? 'degree' : 'degrees'; + return `${i} ${degreeOrDegrees} away`; + } + }); + + return ( + + {labels.map((label, i) => ( + + + {label} + + ))} + + ); +}; + export const ResultsGraph: React.FC<{ readonly paths: readonly WikipediaPageId[][]; readonly pagesById: Record; }> = ({paths, pagesById}) => { const graphRef = useRef(null); + const simulationRef = useRef | null>(null); + const graphSvgRef = useRef | null>(null); + const zoomableRef = useRef | null>(null); const graphWrapperSizeRef = useRef(null); const color = d3.scaleOrdinal(d3.schemeCategory10); @@ -167,9 +202,15 @@ export const ResultsGraph: React.FC<{ } if (previousPageId) { + const sourceNode = nodesData.find((n) => n.id === previousPageId); + const targetNode = nodesData.find((n) => n.id === currentPageId); + if (!sourceNode || !targetNode) { + console.error('Failed to find source or target node'); + return; + } linksData.push({ - source: previousPageId, - target: currentPageId, + source: sourceNode, + target: targetNode, }); } @@ -180,87 +221,182 @@ export const ResultsGraph: React.FC<{ return {nodesData, linksData}; }, [pagesById, paths]); + const zoom = useMemo(() => { + return d3.zoom().on('zoom', (event) => { + if (!graphSvgRef.current) return; + graphSvgRef.current.attr( + `transform`, + `translate(${event.transform.x}, ${event.transform.y}) scale(${event.transform.k})` + ); + }); + }, []); + + const resetGraph = useCallback(() => { + const graphWidth = getGraphWidth(); + + // Reset the center of the simulation force and restart it. + simulationRef.current?.force( + 'center', + d3.forceCenter(graphWidth / 2, DEFAULT_CHART_HEIGHT / 2) + ); + simulationRef.current?.alpha(0.3).restart(); + + // Reset any zoom and pan. + zoomableRef.current?.transition().duration(750).call(zoom.transform, d3.zoomIdentity); + }, [getGraphWidth, zoom]); + const simulation = useMemo(() => { + const {nodesData} = getGraphData(); + return d3.forceSimulation(nodesData); + }, [getGraphData]); + + useEffect(() => { + if (!graphRef.current) return; + + zoomableRef.current = d3 + .select(graphRef.current as Element) + .attr('width', '100%') + .attr('height', '100%') + .call(zoom); + if (!zoomableRef.current) return; + + graphSvgRef.current = zoomableRef.current.append('g'); + if (!graphSvgRef.current) return; + + const pathsLength = paths[0].length; + const targetPageId = pagesById[paths[0][pathsLength - 1]].id; + const {nodesData, linksData} = getGraphData(); - return d3 + + // Define direction arrows. + const defs = graphSvgRef.current.append('defs'); + [ + {id: 'arrow', refX: 18}, + {id: 'arrow-end', refX: 22}, + ].forEach(({id, refX}) => { + defs + .append('marker') + .attr('id', id) + .attr('viewBox', '0 -5 10 10') + .attr('refX', refX) + .attr('refY', 0) + .attr('markerWidth', 8) + .attr('markerHeight', 8) + .attr('orient', 'auto') + .append('svg:path') + .attr('d', 'M0,-5L10,0L0,5'); + }); + + // Insert node labels. + const nodeLabels = graphSvgRef.current + .append('g') + .attr('class', 'node-labels') + .selectAll('text') + .data(nodesData) + .enter() + .append('text') + .attr('x', (d) => { + if (d.degree === 0 || d.degree === pathsLength - 1) { + return 14; + } else { + return 10; + } + }) + .attr('y', 4) + .text((d) => d.title); + + // Insert links between nodes. + const links = graphSvgRef.current + .append('g') + .attr('class', 'links') + .selectAll('line') + .data(linksData) + .enter() + .append('line') + .attr('fill', 'none') + .attr('marker-end', (d) => + // Use a different arrow marker for links to the target page since it has a larger radius. + d.target.id === targetPageId ? 'url(#arrow-end)' : 'url(#arrow)' + ); + + // Insert nodes. Do this after insert links so the nodes sit on top of the links. + const nodes = graphSvgRef.current + .append('g') + .attr('class', 'nodes') + .selectAll('circle') + .data(nodesData) + .enter() + .append('circle') + .attr('r', (d) => (d.degree === 0 || d.degree === pathsLength - 1 ? 10 : 6)) + .attr('fill', (d) => color(d.degree.toString())) + .attr('stroke', (d) => d3.rgb(color(d.degree.toString())).darker(2).toString()) + .on('click', (_, node) => { + // Open Wikipedia page when node is clicked. + window.open(getWikipediaPageUrl(node.title), '_blank'); + }) + .call( + d3 + .drag() + .on('start', (event, d) => { + if (!event.active) { + simulationRef.current?.alphaTarget(0.3).restart(); + } + d.fx = event.x; + d.fy = event.y; + }) + .on('drag', (event, d) => { + d.fx = event.x; + d.fy = event.y; + }) + .on('end', (event, d) => { + if (!event.active) { + simulationRef.current?.alphaTarget(0); + } + d.fx = null; + d.fy = null; + }) + ); + + // Define a force simulation to position nodes and links. + simulationRef.current = d3 .forceSimulation(nodesData) .force( 'link', - d3.forceLink(linksData).id((d: any) => d.id) + d3.forceLink(linksData).id((d) => d.id) ) .force('charge', d3.forceManyBody().strength(-300).distanceMax(500)) .force('center', d3.forceCenter(getGraphWidth() / 2, DEFAULT_CHART_HEIGHT / 2)) .on('tick', () => { - simulation.nodes().forEach((node: any) => { - const nodeElement = d3.select(`#node-${node.id}`); - const linkElement = d3.select(`#link-${node.id}`); - nodeElement.attr('cx', node.x).attr('cy', node.y); - linkElement.attr('x1', node.x).attr('y1', node.y); - }); + // Update element positions on each tick. + nodes.attr('cx', (d) => (d.x ?? 0) + 0).attr('cy', (d) => (d.y ?? 0) + 0); + nodeLabels.attr('transform', (d) => `translate(${d.x}, ${d.y})`); + links + .attr('x1', (d) => d.source.x ?? 0) + .attr('y1', (d) => d.source.y ?? 0) + .attr('x2', (d) => d.target.x ?? 0) + .attr('y2', (d) => d.target.y ?? 0); }); - }, [getGraphData, getGraphWidth]); - useEffect(() => { - const handleResize = debounce((event: UIEvent) => { - if (simulation) { - simulation.force('center', d3.forceCenter(getGraphWidth() / 2, DEFAULT_CHART_HEIGHT / 2)); - simulation.alpha(0.3).restart(); - } - }, 350); - - window.addEventListener('resize', handleResize as EventListener); - - // const handleResize = debounce(() => { - // const newWidth = getGraphWidth(); - // setGraphWidth(newWidth); - // sim.force('center', d3.forceCenter(newWidth / 2, DEFAULT_CHART_HEIGHT / 2)); - // sim.alpha(0.3).restart(); - // }, 350); - // window.addEventListener('resize', handleResize); + // Recenter on window resize. + const handleResizeDebounced = debounce(resetGraph, 350); + window.addEventListener('resize', handleResizeDebounced as EventListener); return () => { - window.removeEventListener('resize', handleResize as EventListener); + graphSvgRef.current?.selectAll('*').remove(); + window.removeEventListener('resize', handleResizeDebounced as EventListener); }; - }, [paths, pagesById, getGraphData, simulation, getGraphWidth]); - - const renderLegend = () => { - const labels = map(range(0, paths[0].length), (i) => { - if (i === 0 && paths[0].length === 1) { - return 'Start / end page'; - } else if (i === 0) { - return 'Start page'; - } else if (i === paths[0].length - 1) { - return 'End page'; - } else { - const degreeOrDegrees = i === 1 ? 'degree' : 'degrees'; - return `${i} ${degreeOrDegrees} away`; - } - }); - return ( - - {labels.map((label, i) => ( - - - {label} - - ))} - - ); - }; + }, [simulation, paths, pagesById, color, getGraphData, getGraphWidth, resetGraph, zoom]); return ( - {renderLegend()} +

Drag to pan. Scroll to zoom.

Click node to open Wikipedia page.

 - simulation?.restart()}> + diff --git a/website/src/components/ResultsList.tsx b/website/src/components/ResultsList.tsx index 8cfe00a..54b239c 100644 --- a/website/src/components/ResultsList.tsx +++ b/website/src/components/ResultsList.tsx @@ -4,7 +4,7 @@ import LazyLoad from 'react-lazyload'; import styled from 'styled-components'; import defaultPageThumbnail from '../images/defaultPageThumbnail.png'; -import {WikipediaPage, WikipediaPageId} from '../types'; +import {WikipediaPage, WikipediaPageId} from '../types.ts'; const ResultsListWrapper = styled.div` margin: 0 auto; diff --git a/website/src/components/blog/posts/SearchResultsAnalysisPost/data.js b/website/src/components/blog/posts/SearchResultsAnalysisPost/data.js index 8124eea..58a0c20 100644 --- a/website/src/components/blog/posts/SearchResultsAnalysisPost/data.js +++ b/website/src/components/blog/posts/SearchResultsAnalysisPost/data.js @@ -1,6 +1,6 @@ import React from 'react'; -import {getNumberWithCommas, getWikipediaPageUrl} from '../../../../utils'; +import {getNumberWithCommas, getWikipediaPageUrl} from '../../../../utils.ts'; import {StyledTextLink} from '../../../common/StyledTextLink.tsx'; export const totalSearches = 503498; diff --git a/website/src/components/blog/posts/SearchResultsAnalysisPost/index.js b/website/src/components/blog/posts/SearchResultsAnalysisPost/index.js index 5acc282..ba0e757 100644 --- a/website/src/components/blog/posts/SearchResultsAnalysisPost/index.js +++ b/website/src/components/blog/posts/SearchResultsAnalysisPost/index.js @@ -1,7 +1,7 @@ import React from 'react'; import {Helmet} from 'react-helmet'; -import {getNumberWithCommas} from '../../../../utils'; +import {getNumberWithCommas} from '../../../../utils.ts'; import {BarChart} from '../../../charts/BarChart'; import {Table} from '../../../charts/Table'; import {StyledTextLink} from '../../../common/StyledTextLink.tsx'; diff --git a/website/src/resources/constants.js b/website/src/resources/constants.ts similarity index 100% rename from website/src/resources/constants.js rename to website/src/resources/constants.ts diff --git a/website/src/types.ts b/website/src/types.ts index dbd7ce1..4de6e7c 100644 --- a/website/src/types.ts +++ b/website/src/types.ts @@ -1,6 +1,7 @@ export type WikipediaPageId = number; export interface WikipediaPage { + readonly id: WikipediaPageId; readonly title: string; readonly url: string; readonly description?: string; diff --git a/website/src/utils.js b/website/src/utils.ts similarity index 73% rename from website/src/utils.js rename to website/src/utils.ts index 831bf48..6b79148 100644 --- a/website/src/utils.js +++ b/website/src/utils.ts @@ -3,14 +3,14 @@ import clone from 'lodash/clone'; import pageTitles from './resources/pageTitles.json'; import wikipediaFacts from './resources/wikipediaFacts.json'; -export const getWikipediaPageUrl = (pageTitle) => { +export const getWikipediaPageUrl = (pageTitle: string): string => { const baseUrl = 'https://en.wikipedia.org/wiki/'; const sanitizedPageTitle = pageTitle.replace(/ /g, '_'); return `${baseUrl}${encodeURIComponent(sanitizedPageTitle)}`; }; -let unusedPageTitles = []; -export const getRandomPageTitle = () => { +let unusedPageTitles: string[] = []; +export const getRandomPageTitle = (): string => { if (unusedPageTitles.length === 0) { unusedPageTitles = clone(pageTitles); } @@ -19,8 +19,8 @@ export const getRandomPageTitle = () => { return unusedPageTitles.splice(indexToRemove, 1)[0]; }; -let unusedWikipediaFacts = []; -export const getRandomWikipediaFact = () => { +let unusedWikipediaFacts: string[] = []; +export const getRandomWikipediaFact = (): string => { if (unusedWikipediaFacts.length === 0) { unusedWikipediaFacts = clone(wikipediaFacts); } @@ -29,6 +29,6 @@ export const getRandomWikipediaFact = () => { return unusedWikipediaFacts.splice(indexToRemove, 1)[0]; }; -export const getNumberWithCommas = (val) => { +export const getNumberWithCommas = (val: number): string => { return val.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ','); };