README.qmd

---
format: gfm
title: "Notes from a hackathon"
---

Notes from a hackathon.

# Introduction

This is a collection of notes from a hackathon held at the Alan Turing Institute in London on 2023-04-20.

# Setup

## Getting the code repos

The starting point was two code repos:

- Front end (note the use of PMTiles): https://github.com/ADD-William-WaltersDavis/planning_tool
- Backend code in Rust to get scores and routes for each square: https://github.com/adam-jb/rust_connectivity_pt_tiles

These repos were cloned from GitHub as follows:

```{bash}
#| eval: false
gh repo clone ADD-William-WaltersDavis/planning_tool
gh repo clone adam-jb/rust_connectivity_pt_tiles

# add the submodules
git submodule add https://github.com/ADD-William-WaltersDavis/planning_tool planning_tool
git submodule add https://github.com/adam-jb/rust_connectivity_pt_tiles rust_connectivity_pt_tiles
```

## Visualising the data


```{bash}
ls planning_tool
```

Run the front end:

```{bash}
#| eval: false
cd planning_tool
npm install
# global install of vite:
sudo npm install -g vite
npm run dev
```

```{bash}
```


# Working with link data

A starting point was files prepared for the event.
These were copied to the route of the project with bash as follows:

```{bash}
#| eval: false
mkdir data
mv -v ~/Downloads/*.pickle data
```

```{python}
#| eval: false
#| echo: false
# move all pickle files from ~/Downloads to ./data:
import os
import shutil
# Create the data directory if it doesn't exist:
if not os.path.exists('./data'):
    os.makedirs('./data')
for file in os.listdir('~/Downloads'):
    if file.endswith('.pickle'):
        shutil.move(os.path.join('~/Downloads', file), './data')
```

We can list the pickle files as follows:

```{python}
import os
# List pickle files in route directory:
for file in os.listdir('data'):
    print(file)
```


```{python}
import pickle
# Read the first pickle file:
with open('data/AA_example_links.pickle', 'rb') as f:
    links = pickle.load(f)
```

Show what's in the links object, with output showning first 80 characters:

```{python}
# Find length of links:
len(links)
links.__class__
links.__sizeof__()
links_items = links.items()
links_items.__class__
# links_items[:10]
# Convert dict to list:
links_list = list(links_items)
links_list.__class__
len(links_list)
# Convert list to character string:
links_str = str(links_list)
links_str[:80]

```

We converted the object to json as follows:

```{python}
# Define a function that converts the dict to json and save the output to a file:
import json
def write_json(data, filename='data/AA_example_links.json'):
    with open(filename,'w') as f:
        json.dump(data, f, indent=4)

write_json(links, 'data/AA_example_links.json')
```

Test reading as a GeoJSON file:

```{python}
#| eval: false
import geopandas as gpd
# The following fails with error:
gdf = gpd.read_file('data/AA_example_links.json')
```

## Read and visualise with R

We'll use the following packages:

```{r}
#| message: false
library(sf)
library(tidyverse)
library(tmap)
tmap_mode("view")
# Install mastermapr dependency:
remotes::install_github("acteng/mastermapr")
```

```{r}
```

```{r}
gdf_list = jsonlite::read_json("data/AA_example_links.json")
str(gdf_list[[1]][[1]])
length(gdf_list)
# show 1st element:
length(gdf_list[[1]])
gdf_list[[1]][[1]]
# create geographic representation of first file:
gdf_origin_coords = c(
    gdf_list[[1]][[1]][[8]][[1]],
    gdf_list[[1]][[1]][[8]][[2]]
    )
gdf_destination_coords = c(
    gdf_list[[1]][[1]][[9]][[1]],
    gdf_list[[1]][[1]][[9]][[2]]
    )
gdf_matrix = rbind(gdf_origin_coords, gdf_destination_coords)
gdf_linestring = sf::st_linestring(gdf_matrix)
sfc_linestring = sf::st_sfc(gdf_linestring)
sf_linestring = sf::st_as_sf(sfc_linestring)
qtm(sf_linestring)
```

## Iterate for all links and visualise network

First we'll generalise the previous code to a function:

```{r}
link_coordinates = function(link) {
    # get origin and destination coordinates:
    gdf_origin_coords = c(
        link[[8]][[1]],
        link[[8]][[2]]
        )
    gdf_destination_coords = c(
        link[[9]][[1]],
        link[[9]][[2]]
        )
    # create matrix of coordinates:
    gdf_matrix = rbind(gdf_origin_coords, gdf_destination_coords)
    # create linestring:
    gdf_linestring = sf::st_linestring(gdf_matrix)
    # create sfc:
    sfc_linestring = sf::st_sfc(gdf_linestring)
    # create sf:
    sf_linestring = sf::st_as_sf(sfc_linestring)
    return(sf_linestring)
}
# Test the function with the first link:
link_coordinates(gdf_list[[1]][[1]])
links = gdf_list[[1]]

links_to_sfc = function(links) {
    list_linstrings = lapply(links, link_coordinates)
    # class(list_linstrings)
    # qtm(list_linstrings[[2]])
    # combine the sf linstrings into a single object:
    # Note: inefficient implemenation TODO, make more efficient:
    # sf_links = do.call(rbind, list_linstrings)
    sf_links = mastermapr::fastrbindsf(list_linstrings)
    return(sf_links)
}
links_sf = links_to_sfc(gdf_list[[1]])
class(links_sf)
qtm(links_sf)
```

## Get attribute data for links

```{r}
# Function to get link attributes
# For testing:
# link = gdf_list[[1]][[1]]
link_attributes = function(
    link,
    attribute_names = c(
        'Business',
        'Education',
        'Entertainment',
        'Shopping',
        'Visit friends',
        'start_node',
        'end_node',
        'start_longlat',
        'end_longlat'
        )
    ) {
    # get origin and destination coordinates:
    link_id = link[[1]]
    link_length = link[[2]]
    # Subset first 7 elements of list:
    link_attributes = link[1:7]
    # Create a data frame:
    link_df = data.frame(link_attributes)
    names(link_df) = attribute_names[1:7]
    return(link_df)
}
# Test on a single link:
link_attributes(gdf_list[[1]][[1]])

links_to_df = function(links) {
    list_dfs = pbapply::pblapply(links, link_attributes)
    data.table::rbindlist(list_dfs)
}

# Function to get link attributes for all links:
links_to_sf = function(links, subset_on = NULL, keep_n = Inf, keep_percent = 100, pt_threshold = 100326060) {
    links_df = links_to_df(links)
    if(!is.null(subset_on)) {
        # indices of top keep_n items:
        if(keep_n < nrow(links_df)) {
            indices = order(links_df[[subset_on]],  decreasing = TRUE)
            indices = indices[seq(keep_n)]
            links_df = links_df[indices, ]
        }
        links_sfc = links_to_sfc(links[indices])
    } else {
        links_sfc = links_to_sfc(links)
    }
    # Create a spatial data frame:
    # class(links_sfc$x)
    links_sf = sf::st_sf(geometry = sf::st_geometry(links_sfc), links_df, crs = 4326)
    links_sf$Mode = dplyr::case_when(
        links_sf$start_node > pt_threshold ~ "Public Transport",
        # TODO: add more rules
        TRUE ~ "Walking"
    )
    links_sf
}
links_sf = links_to_sf(gdf_list[[1]], keep_n = 1000, subset_on = "Business")
names(links_sf)
nrow(links_sf)
links_sf |>
  select(Business:`Visit friends`) |>
  sf::st_drop_geometry() |>
  summary()
```

## Style the network

We can now visualise the network with the `tmap` package:


```{r networkvis}
links_sf |>
  mutate(across(Business:`Visit friends`, sqrt)) |>
  tm_shape() +
  tm_lines(lwd = "Business", scale = 15, col = "Mode", palette = "Set1") +
  tm_scale_bar() 
```

```{r rnetbusiness}
table(links_sf$Mode)
links_sf |>
  mutate(across(Business:`Visit friends`, sqrt)) |>
  tm_shape() +
  tm_lines(lwd = "Visit friends", scale = 15, col = "Mode", palette = "Set1") +
  tm_scale_bar() 
```

```{r rnetfriends}
links_sf_friends = links_to_sf(gdf_list[[1]], keep_n = 1000, subset_on = "Visit friends")
table(links_sf_friends$Mode)
links_sf_friends |>
  mutate(across(Business:`Visit friends`, sqrt)) |>
  tm_shape() +
  tm_lines(lwd = "Visit friends", scale = 15, col = "Mode", palette = "Set1") +
  tm_scale_bar() 
```

```{r rneteducation}
links_sf_education = links_to_sf(gdf_list[[1]], keep_n = 1000, subset_on = "Education")
table(links_sf_education$Mode)
links_sf_education |>
  mutate(across(Business:`Visit friends`, sqrt)) |>
  tm_shape() +
  tm_lines(lwd = "Education", scale = 15, col = "Mode", palette = "Set1") +
  tm_scale_bar() 
```

```{python}
```


```{python}
#| eval: false
#| echo: false
# Print first 10 characters of links:
links[:10]
# Unhashable type: 'slice'
```

```{python}
```

```{python}
```