<!DOCTYPE html>
<html>
<head>
<style type="text/css">
.knitr .inline {
background-color: #f7f7f7;
border:solid 1px #B0B0B0;
}
.error {
font-weight: bold;
color: #FF0000;
}
.warning {
font-weight: bold;
}
.message {
font-style: italic;
}
.source, .output, .warning, .error, .message {
padding: 0 1em;
border:solid 1px #F7F7F7;
}
.source {
background-color: #f5f5f5;
}
.rimage .left {
text-align: left;
}
.rimage .right {
text-align: right;
}
.rimage .center {
text-align: center;
}
.hl.num {
color: #AF0F91;
}
.hl.str {
color: #317ECC;
}
.hl.com {
color: #AD95AF;
font-style: italic;
}
.hl.opt {
color: #000000;
}
.hl.std {
color: #585858;
}
.hl.kwa {
color: #295F94;
font-weight: bold;
}
.hl.kwb {
color: #B05A65;
}
.hl.kwc {
color: #55aa55;
}
.hl.kwd {
color: #BC5A65;
font-weight: bold;
}
</style>
<script src="https://yihui.name/media/js/center-images.js"></script>
<title>A Report Generated by knitr</title>
</head>
<body>
<p>This report is automatically generated with the R
package <a href="https://yihui.name/knitr/"><strong>knitr</strong></a>
(version <code class="knitr inline">1.20</code>)
.</p>
<div class="chunk" id="auto-report"><div class="rcode"><div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(lmerTest)</span>
</pre></div>
<div class="message"><pre class="knitr r">## Loading required package: lme4
</pre></div>
<div class="message"><pre class="knitr r">## Loading required package: Matrix
</pre></div>
<div class="message"><pre class="knitr r">##
## Attaching package: 'lmerTest'
</pre></div>
<div class="message"><pre class="knitr r">## The following object is masked from 'package:lme4':
##
## lmer
</pre></div>
<div class="message"><pre class="knitr r">## The following object is masked from 'package:stats':
##
## step
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(lsmeans)</span>
</pre></div>
<div class="message"><pre class="knitr r">## The 'lsmeans' package is being deprecated.
## Users are encouraged to switch to 'emmeans'.
## See help('transition') for more information, including how
## to convert 'lsmeans' objects and scripts to work with 'emmeans'.
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(scales)</span>
<span class="hl kwd">library</span><span class="hl std">(ggrepel)</span>
</pre></div>
<div class="message"><pre class="knitr r">## Loading required package: ggplot2
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(readr)</span>
</pre></div>
<div class="message"><pre class="knitr r">##
## Attaching package: 'readr'
</pre></div>
<div class="message"><pre class="knitr r">## The following object is masked from 'package:scales':
##
## col_factor
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(magrittr)</span>
<span class="hl kwd">library</span><span class="hl std">(dplyr)</span>
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: package 'dplyr' was built under R version 3.5.2
</pre></div>
<div class="message"><pre class="knitr r">##
## Attaching package: 'dplyr'
</pre></div>
<div class="message"><pre class="knitr r">## The following objects are masked from 'package:stats':
##
## filter, lag
</pre></div>
<div class="message"><pre class="knitr r">## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(tidyr)</span>
</pre></div>
<div class="message"><pre class="knitr r">##
## Attaching package: 'tidyr'
</pre></div>
<div class="message"><pre class="knitr r">## The following object is masked from 'package:magrittr':
##
## extract
</pre></div>
<div class="message"><pre class="knitr r">## The following object is masked from 'package:Matrix':
##
## expand
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(ggplot2)</span>
<span class="hl kwd">library</span><span class="hl std">(xtable)</span>
<span class="hl kwd">library</span><span class="hl std">(emmeans)</span>
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: package 'emmeans' was built under R version 3.5.2
</pre></div>
<div class="message"><pre class="knitr r">##
## Attaching package: 'emmeans'
</pre></div>
<div class="message"><pre class="knitr r">## The following objects are masked from 'package:lsmeans':
##
## .all.vars, .aovlist.dffun, .diag, .get.offset, .my.vcov, add_grouping,
## as.glht, contrast, get.lsm.option, lsm, lsm.options, lsmeans, lsmip, lsmobj,
## lstrends, make.tran, pmm, pmmeans, pmmip, pmmobj, pmtrends, regrid, test
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">library</span><span class="hl std">(lsr)</span>
<span class="hl kwd">theme_set</span><span class="hl std">(</span><span class="hl kwd">theme_bw</span><span class="hl std">())</span>
<span class="hl kwd">emm_options</span><span class="hl std">(</span><span class="hl kwc">pbkrtest.limit</span> <span class="hl std">=</span> <span class="hl num">12500</span><span class="hl std">)</span>
<span class="hl kwd">emm_options</span><span class="hl std">(</span><span class="hl kwc">lmerTest.limit</span> <span class="hl std">=</span> <span class="hl num">12500</span><span class="hl std">)</span>
<span class="hl std">my_pallete</span> <span class="hl kwb"><-</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl str">'#92c5de'</span><span class="hl std">,</span><span class="hl str">'#0571b0'</span><span class="hl std">,</span><span class="hl str">'#f4a582'</span><span class="hl std">,</span><span class="hl str">'#ca0020'</span><span class="hl std">,</span><span class="hl str">'#bababa'</span><span class="hl std">)</span>
<span class="hl kwd">source</span><span class="hl std">(</span><span class="hl str">'lib.R'</span><span class="hl std">)</span>
</pre></div>
<div class="message"><pre class="knitr r">##
## Attaching package: 'ggridges'
</pre></div>
<div class="message"><pre class="knitr r">## The following object is masked from 'package:ggplot2':
##
## scale_discrete_manual
</pre></div>
<div class="message"><pre class="knitr r">## ── Attaching packages ───────────────────────────────────────────────── tidyverse 1.2.1 ──
</pre></div>
<div class="message"><pre class="knitr r">## ✔ tibble 2.1.3 ✔ stringr 1.3.1
## ✔ purrr 0.2.5 ✔ forcats 0.3.0
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: package 'tibble' was built under R version 3.5.2
</pre></div>
<div class="message"><pre class="knitr r">## ── Conflicts ──────────────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ readr::col_factor() masks scales::col_factor()
## ✖ purrr::discard() masks scales::discard()
## ✖ tidyr::expand() masks Matrix::expand()
## ✖ tidyr::extract() masks magrittr::extract()
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ✖ ggridges::scale_discrete_manual() masks ggplot2::scale_discrete_manual()
## ✖ purrr::set_names() masks magrittr::set_names()
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwa">if</span> <span class="hl std">(</span><span class="hl opt">!</span><span class="hl kwd">dir.exists</span><span class="hl std">(</span><span class="hl str">'figures/exp3'</span><span class="hl std">))</span>
<span class="hl kwd">dir.create</span><span class="hl std">(</span><span class="hl str">'figures/exp3'</span><span class="hl std">,</span> <span class="hl kwc">recursive</span> <span class="hl std">= T)</span>
<span class="hl com"># read data</span>
<span class="hl std">woa_responses</span> <span class="hl kwb"><-</span> <span class="hl kwd">read_csv</span><span class="hl std">(</span><span class="hl str">"data/exp3/data.csv"</span><span class="hl std">)</span>
</pre></div>
<div class="message"><pre class="knitr r">## Parsed with column specification:
## cols(
## worker_id = col_character(),
## condition = col_character(),
## q_id = col_integer(),
## model_pred = col_double(),
## actual_price = col_double(),
## user_init_pred = col_double(),
## final_pred = col_double(),
## step1_time = col_integer(),
## step2_time = col_integer()
## )
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># recode conditions and compute outcome measures</span>
<span class="hl std">woa_responses</span> <span class="hl kwb"><-</span> <span class="hl std">woa_responses</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">condition</span> <span class="hl std">=</span> <span class="hl kwd">recode_factor</span><span class="hl std">(</span><span class="hl kwd">as.factor</span><span class="hl std">(condition),</span>
<span class="hl str">"C1"</span> <span class="hl std">=</span> <span class="hl str">"CLEAR-2"</span><span class="hl std">,</span>
<span class="hl str">"C3"</span> <span class="hl std">=</span> <span class="hl str">"CLEAR-8"</span><span class="hl std">,</span>
<span class="hl str">"C0"</span> <span class="hl std">=</span> <span class="hl str">"BB-2"</span><span class="hl std">,</span>
<span class="hl str">"C2"</span> <span class="hl std">=</span> <span class="hl str">"BB-8"</span><span class="hl std">,</span>
<span class="hl str">"C4"</span> <span class="hl std">=</span> <span class="hl str">"EXPERT"</span><span class="hl std">))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">pred_err</span> <span class="hl std">=</span> <span class="hl kwd">abs</span><span class="hl std">(final_pred</span> <span class="hl opt">-</span> <span class="hl std">actual_price),</span>
<span class="hl kwc">deviation</span> <span class="hl std">=</span> <span class="hl kwd">abs</span><span class="hl std">(final_pred</span> <span class="hl opt">-</span> <span class="hl std">model_pred),</span>
<span class="hl kwc">woa</span> <span class="hl std">=</span> <span class="hl kwd">abs</span><span class="hl std">(final_pred</span> <span class="hl opt">-</span> <span class="hl std">user_init_pred)</span> <span class="hl opt">/</span> <span class="hl kwd">abs</span><span class="hl std">(model_pred</span> <span class="hl opt">-</span> <span class="hl std">user_init_pred))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">ungroup</span><span class="hl std">()</span>
<span class="hl com">#########################################</span>
<span class="hl com"># histograms</span>
<span class="hl com">#########################################</span>
<span class="hl com"># users' final prediction</span>
<span class="hl std">breaks</span> <span class="hl kwb"><-</span> <span class="hl kwd">seq</span><span class="hl std">(</span><span class="hl num">0</span><span class="hl std">,</span><span class="hl num">3</span><span class="hl std">,</span><span class="hl num">1</span><span class="hl std">)</span>
<span class="hl std">responses1</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(woa_responses, q_id</span> <span class="hl opt"><</span> <span class="hl num">6</span><span class="hl std">)</span>
<span class="hl std">responses1</span><span class="hl opt">$</span><span class="hl std">q_id</span> <span class="hl kwb"><-</span> <span class="hl std">responses1</span><span class="hl opt">$</span><span class="hl std">q_id</span> <span class="hl opt">+</span> <span class="hl num">1</span>
<span class="hl std">responses2</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(woa_responses, q_id</span> <span class="hl opt">>=</span> <span class="hl num">6</span><span class="hl std">)</span>
<span class="hl std">responses2</span><span class="hl opt">$</span><span class="hl std">q_id</span> <span class="hl kwb"><-</span> <span class="hl std">responses2</span><span class="hl opt">$</span><span class="hl std">q_id</span> <span class="hl opt">+</span> <span class="hl num">1</span>
<span class="hl com"># replace -1 for actual price of apartment 12 (synthetic) with NA</span>
<span class="hl std">responses1</span><span class="hl opt">$</span><span class="hl std">actual_price</span> <span class="hl kwb"><-</span> <span class="hl kwd">replace</span><span class="hl std">(responses1</span><span class="hl opt">$</span><span class="hl std">actual_price,</span> <span class="hl kwd">which</span><span class="hl std">(responses1</span><span class="hl opt">$</span><span class="hl std">actual_price</span> <span class="hl opt">== -</span><span class="hl num">1</span><span class="hl std">),</span> <span class="hl num">NA</span><span class="hl std">)</span>
<span class="hl std">responses2</span><span class="hl opt">$</span><span class="hl std">actual_price</span> <span class="hl kwb"><-</span> <span class="hl kwd">replace</span><span class="hl std">(responses2</span><span class="hl opt">$</span><span class="hl std">actual_price,</span> <span class="hl kwd">which</span><span class="hl std">(responses2</span><span class="hl opt">$</span><span class="hl std">actual_price</span> <span class="hl opt">== -</span><span class="hl num">1</span><span class="hl std">),</span> <span class="hl num">NA</span><span class="hl std">)</span>
<span class="hl com"># shift plot</span>
<span class="hl std">plot_shifts</span> <span class="hl kwb"><-</span> <span class="hl kwa">function</span><span class="hl std">(</span><span class="hl kwc">response_data</span><span class="hl std">){</span>
<span class="hl com"># summary stats</span>
<span class="hl std">plot_data</span> <span class="hl kwb"><-</span> <span class="hl std">response_data</span> <span class="hl opt">%>%</span>
<span class="hl kwd">group_by</span><span class="hl std">(condition, q_id)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">summarize</span><span class="hl std">(</span><span class="hl kwc">mean_user_init_pred</span> <span class="hl std">=</span> <span class="hl kwd">mean</span><span class="hl std">(user_init_pred),</span>
<span class="hl kwc">mean_final_pred</span> <span class="hl std">=</span> <span class="hl kwd">mean</span><span class="hl std">(final_pred))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">group_by</span><span class="hl std">(q_id)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">arrange</span><span class="hl std">(q_id, condition)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">y</span> <span class="hl std">=</span> <span class="hl num">1</span><span class="hl opt">:</span><span class="hl kwd">n</span><span class="hl std">())</span>
<span class="hl std">breaks</span> <span class="hl kwb"><-</span> <span class="hl kwd">seq</span><span class="hl std">(</span><span class="hl num">0</span><span class="hl std">,</span><span class="hl num">3</span><span class="hl std">,</span><span class="hl num">1</span><span class="hl std">)</span>
<span class="hl std">response_data</span> <span class="hl opt">%>%</span>
<span class="hl kwd">group_by</span><span class="hl std">(condition, q_id)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">filter</span><span class="hl std">(user_init_pred</span> <span class="hl opt">>=</span> <span class="hl kwd">quantile</span><span class="hl std">(user_init_pred,</span> <span class="hl num">0.05</span><span class="hl std">),</span>
<span class="hl std">user_init_pred</span> <span class="hl opt"><=</span> <span class="hl kwd">quantile</span><span class="hl std">(user_init_pred,</span> <span class="hl num">0.95</span><span class="hl std">),</span>
<span class="hl std">final_pred</span> <span class="hl opt">>=</span> <span class="hl kwd">quantile</span><span class="hl std">(final_pred,</span> <span class="hl num">0.05</span><span class="hl std">),</span>
<span class="hl std">final_pred</span> <span class="hl opt"><=</span> <span class="hl kwd">quantile</span><span class="hl std">(final_pred,</span> <span class="hl num">0.95</span><span class="hl std">))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">ungroup</span><span class="hl std">()</span> <span class="hl opt">%>%</span>
<span class="hl kwd">ggplot</span><span class="hl std">(</span><span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">x</span> <span class="hl std">= final_pred,</span> <span class="hl kwc">fill</span> <span class="hl std">= condition,</span> <span class="hl kwc">color</span> <span class="hl std">= condition))</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_histogram</span><span class="hl std">(</span><span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">x</span> <span class="hl std">= user_init_pred),</span> <span class="hl kwc">alpha</span> <span class="hl std">=</span> <span class="hl num">0.2</span><span class="hl std">,</span> <span class="hl kwc">linetype</span> <span class="hl std">=</span> <span class="hl str">'dotted'</span><span class="hl std">,</span> <span class="hl kwc">binwidth</span> <span class="hl std">=</span> <span class="hl num">0.1</span><span class="hl std">)</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_histogram</span><span class="hl std">(</span><span class="hl kwc">alpha</span> <span class="hl std">=</span> <span class="hl num">0.5</span><span class="hl std">,</span> <span class="hl kwc">binwidth</span> <span class="hl std">=</span> <span class="hl num">0.1</span><span class="hl std">)</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_point</span><span class="hl std">(</span><span class="hl kwc">data</span> <span class="hl std">= plot_data,</span> <span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">x</span> <span class="hl std">= mean_final_pred,</span> <span class="hl kwc">y</span> <span class="hl std">=</span> <span class="hl num">80</span><span class="hl std">),</span> <span class="hl kwc">shape</span> <span class="hl std">=</span> <span class="hl num">20</span><span class="hl std">)</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_point</span><span class="hl std">(</span><span class="hl kwc">data</span> <span class="hl std">= plot_data,</span> <span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">x</span> <span class="hl std">= mean_user_init_pred,</span> <span class="hl kwc">y</span> <span class="hl std">=</span> <span class="hl num">80</span><span class="hl std">),</span> <span class="hl kwc">shape</span> <span class="hl std">=</span> <span class="hl num">1</span><span class="hl std">)</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_segment</span><span class="hl std">(</span><span class="hl kwc">data</span> <span class="hl std">= plot_data,</span>
<span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">x</span> <span class="hl std">= mean_user_init_pred,</span> <span class="hl kwc">y</span> <span class="hl std">=</span> <span class="hl num">95</span><span class="hl std">,</span>
<span class="hl kwc">xend</span> <span class="hl std">= mean_final_pred,</span> <span class="hl kwc">yend</span> <span class="hl std">=</span> <span class="hl num">95</span><span class="hl std">),</span>
<span class="hl kwc">arrow</span> <span class="hl std">=</span> <span class="hl kwd">arrow</span><span class="hl std">(</span><span class="hl kwc">length</span> <span class="hl std">=</span> <span class="hl kwd">unit</span><span class="hl std">(</span><span class="hl num">0.03</span><span class="hl std">,</span> <span class="hl str">"npc"</span><span class="hl std">)))</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_vline</span><span class="hl std">(</span> <span class="hl kwc">alpha</span> <span class="hl std">=</span> <span class="hl num">.2</span><span class="hl std">,</span> <span class="hl kwc">size</span> <span class="hl std">=</span> <span class="hl num">.5</span><span class="hl std">,</span> <span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">xintercept</span> <span class="hl std">= model_pred,</span> <span class="hl kwc">linetype</span> <span class="hl std">=</span> <span class="hl str">"Model's prediction"</span><span class="hl std">))</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_vline</span><span class="hl std">(</span><span class="hl kwc">alpha</span><span class="hl std">=</span><span class="hl num">.2</span><span class="hl std">,</span> <span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">.5</span><span class="hl std">,</span> <span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">xintercept</span> <span class="hl std">= actual_price,</span> <span class="hl kwc">linetype</span><span class="hl std">=</span><span class="hl str">"Actual price"</span><span class="hl std">))</span><span class="hl opt">+</span>
<span class="hl kwd">facet_grid</span><span class="hl std">(condition</span> <span class="hl opt">~</span> <span class="hl std">q_id,</span> <span class="hl kwc">scale</span> <span class="hl std">=</span> <span class="hl str">"free_x"</span><span class="hl std">)</span> <span class="hl opt">+</span>
<span class="hl kwd">scale_x_continuous</span><span class="hl std">(</span><span class="hl kwc">breaks</span> <span class="hl std">= breaks,</span> <span class="hl kwc">labels</span> <span class="hl std">=</span> <span class="hl kwd">sprintf</span><span class="hl std">(</span><span class="hl str">'$%dM'</span><span class="hl std">, breaks))</span> <span class="hl opt">+</span>
<span class="hl kwd">scale_fill_manual</span><span class="hl std">(</span><span class="hl kwc">guide</span> <span class="hl std">=</span> <span class="hl num">FALSE</span><span class="hl std">,</span> <span class="hl kwc">values</span><span class="hl std">=my_pallete)</span> <span class="hl opt">+</span>
<span class="hl kwd">scale_colour_manual</span><span class="hl std">(</span><span class="hl kwc">guide</span> <span class="hl std">=</span> <span class="hl num">FALSE</span><span class="hl std">,</span> <span class="hl kwc">values</span><span class="hl std">=my_pallete)</span> <span class="hl opt">+</span>
<span class="hl kwd">theme</span><span class="hl std">(</span><span class="hl kwc">panel.grid.major</span> <span class="hl std">=</span> <span class="hl kwd">element_blank</span><span class="hl std">(),</span> <span class="hl kwc">panel.grid.minor</span> <span class="hl std">=</span> <span class="hl kwd">element_blank</span><span class="hl std">(),</span>
<span class="hl kwc">legend.position</span><span class="hl std">=</span><span class="hl str">"top"</span><span class="hl std">,</span><span class="hl kwc">legend.title</span> <span class="hl std">=</span> <span class="hl kwd">element_blank</span><span class="hl std">(),</span> <span class="hl kwc">legend.key</span> <span class="hl std">=</span> <span class="hl kwd">element_rect</span><span class="hl std">(</span><span class="hl kwc">size</span> <span class="hl std">=</span> <span class="hl num">10</span><span class="hl std">),</span> <span class="hl kwc">legend.key.size</span> <span class="hl std">=</span> <span class="hl kwd">unit</span><span class="hl std">(</span><span class="hl num">1.5</span><span class="hl std">,</span> <span class="hl str">'lines'</span><span class="hl std">),</span> <span class="hl kwc">legend.text</span><span class="hl std">=</span><span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">12</span><span class="hl std">),</span>
<span class="hl kwc">axis.title.x</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">20</span><span class="hl std">),</span> <span class="hl kwc">axis.title.y</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">18</span><span class="hl std">),</span> <span class="hl kwc">axis.text.x</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">angle</span> <span class="hl std">=</span> <span class="hl num">45</span><span class="hl std">,</span> <span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">10</span><span class="hl std">),</span> <span class="hl kwc">axis.text.y</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">14</span><span class="hl std">),</span>
<span class="hl kwc">strip.text</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">14</span><span class="hl std">))</span><span class="hl opt">+</span>
<span class="hl kwd">labs</span><span class="hl std">(</span><span class="hl kwc">x</span> <span class="hl std">=</span> <span class="hl str">"Participants' initial and final predictions"</span><span class="hl std">,</span>
<span class="hl kwc">y</span> <span class="hl std">=</span> <span class="hl str">"Number of participants"</span><span class="hl std">)</span>
<span class="hl std">}</span>
<span class="hl kwd">plot_shifts</span><span class="hl std">(responses1)</span>
</pre></div>
</div><div class="rimage center"><img src="figure/analyze-experiment-3-Rhtmlauto-report-1.png" title="plot of chunk auto-report" alt="plot of chunk auto-report" class="plot" /></div><div class="rcode">
<div class="source"><pre class="knitr r"><span class="hl kwd">ggsave</span><span class="hl std">(</span><span class="hl kwc">file</span> <span class="hl std">=</span> <span class="hl str">'figures/exp3/shift_histograms_1.pdf'</span><span class="hl std">,</span> <span class="hl kwc">height</span><span class="hl std">=</span><span class="hl num">10</span><span class="hl std">,</span> <span class="hl kwc">width</span><span class="hl std">=</span><span class="hl num">10</span><span class="hl std">)</span>
<span class="hl kwd">plot_shifts</span><span class="hl std">(responses2)</span>
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: Removed 877 rows containing missing values (geom_vline).
</pre></div>
</div><div class="rimage center"><img src="figure/analyze-experiment-3-Rhtmlauto-report-2.png" title="plot of chunk auto-report" alt="plot of chunk auto-report" class="plot" /></div><div class="rcode">
<div class="source"><pre class="knitr r"><span class="hl kwd">ggsave</span><span class="hl std">(</span><span class="hl kwc">file</span> <span class="hl std">=</span> <span class="hl str">'figures/exp3/shift_histograms_2.pdf'</span><span class="hl std">,</span> <span class="hl kwc">height</span><span class="hl std">=</span><span class="hl num">10</span><span class="hl std">,</span> <span class="hl kwc">width</span><span class="hl std">=</span><span class="hl num">10</span><span class="hl std">)</span>
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: Removed 877 rows containing missing values (geom_vline).
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># R2: how many subjects think the model underestimates the sale price of the unusual 3 bathroom apt?</span>
<span class="hl std">woa_responses</span> <span class="hl opt">%>%</span>
<span class="hl kwd">filter</span><span class="hl std">(q_id</span> <span class="hl opt">==</span> <span class="hl num">11</span><span class="hl std">)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">summarize</span><span class="hl std">(</span><span class="hl kwd">mean</span><span class="hl std">(user_init_pred</span> <span class="hl opt"><</span> <span class="hl std">model_pred))</span>
</pre></div>
<div class="output"><pre class="knitr r">## # A tibble: 1 x 1
## `mean(user_init_pred < model_pred)`
## <dbl>
## 1 0.888
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl std">woa_responses</span> <span class="hl opt">%>%</span>
<span class="hl kwd">filter</span><span class="hl std">(q_id</span> <span class="hl opt">==</span> <span class="hl num">11</span><span class="hl std">)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">summarize</span><span class="hl std">(</span><span class="hl kwd">mean</span><span class="hl std">(user_init_pred</span> <span class="hl opt">></span> <span class="hl std">model_pred))</span>
</pre></div>
<div class="output"><pre class="knitr r">## # A tibble: 1 x 1
## `mean(user_init_pred > model_pred)`
## <dbl>
## 1 0.0721
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># R2: How many choose the midpoint of the scale? The ends?</span>
<span class="hl std">breaks</span> <span class="hl kwb"><-</span> <span class="hl kwd">seq</span><span class="hl std">(</span><span class="hl num">0</span><span class="hl std">,</span><span class="hl num">3</span><span class="hl std">,</span><span class="hl num">1</span><span class="hl std">)</span>
<span class="hl std">woa_responses</span> <span class="hl opt">%>%</span>
<span class="hl kwd">count</span><span class="hl std">(user_init_pred)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">frac</span> <span class="hl std">= n</span> <span class="hl opt">/</span> <span class="hl kwd">sum</span><span class="hl std">(n))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">ggplot</span><span class="hl std">(</span><span class="hl kwd">aes</span><span class="hl std">(</span><span class="hl kwc">x</span> <span class="hl std">= user_init_pred,</span> <span class="hl kwc">y</span> <span class="hl std">= frac))</span> <span class="hl opt">+</span>
<span class="hl kwd">scale_x_continuous</span><span class="hl std">(</span><span class="hl kwc">breaks</span> <span class="hl std">= breaks,</span> <span class="hl kwc">labels</span> <span class="hl std">=</span> <span class="hl kwd">sprintf</span><span class="hl std">(</span><span class="hl str">'$%dM'</span><span class="hl std">, breaks))</span> <span class="hl opt">+</span>
<span class="hl kwd">geom_bar</span><span class="hl std">(</span><span class="hl kwc">stat</span> <span class="hl std">=</span> <span class="hl str">"identity"</span><span class="hl std">)</span><span class="hl opt">+</span>
<span class="hl kwd">xlab</span><span class="hl std">(</span><span class="hl str">"Participants' initial prediction"</span><span class="hl std">)</span><span class="hl opt">+</span>
<span class="hl kwd">ylab</span><span class="hl std">(</span><span class="hl str">"Fraction of responses"</span><span class="hl std">)</span><span class="hl opt">+</span>
<span class="hl kwd">theme</span><span class="hl std">(</span><span class="hl kwc">legend.position</span><span class="hl std">=</span><span class="hl str">"bottom"</span><span class="hl std">,</span><span class="hl kwc">axis.title.x</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">14</span><span class="hl std">),</span> <span class="hl kwc">axis.title.y</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">18</span><span class="hl std">),</span>
<span class="hl kwc">axis.text.x</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">9</span><span class="hl std">),</span> <span class="hl kwc">axis.text.y</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">14</span><span class="hl std">),</span>
<span class="hl kwc">legend.text</span><span class="hl std">=</span><span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">9</span><span class="hl std">),</span> <span class="hl kwc">legend.title</span> <span class="hl std">=</span> <span class="hl kwd">element_blank</span><span class="hl std">(),</span> <span class="hl kwc">legend.spacing.x</span><span class="hl std">=</span><span class="hl kwd">unit</span><span class="hl std">(</span><span class="hl num">0.5</span><span class="hl std">,</span><span class="hl str">"line"</span><span class="hl std">),</span> <span class="hl kwc">legend.margin</span><span class="hl std">=</span><span class="hl kwd">margin</span><span class="hl std">(</span><span class="hl kwc">t</span><span class="hl std">=</span><span class="hl opt">-</span><span class="hl num">.5</span><span class="hl std">,</span> <span class="hl kwc">r</span><span class="hl std">=</span><span class="hl num">0</span><span class="hl std">,</span> <span class="hl kwc">b</span><span class="hl std">=</span><span class="hl num">0</span><span class="hl std">,</span> <span class="hl kwc">l</span><span class="hl std">=</span><span class="hl num">0</span><span class="hl std">,</span> <span class="hl kwc">unit</span><span class="hl std">=</span><span class="hl str">"cm"</span><span class="hl std">),</span>
<span class="hl kwc">strip.text</span> <span class="hl std">=</span> <span class="hl kwd">element_text</span><span class="hl std">(</span><span class="hl kwc">size</span><span class="hl std">=</span><span class="hl num">9</span><span class="hl std">))</span><span class="hl opt">+</span>
<span class="hl kwd">ggsave</span><span class="hl std">(</span><span class="hl str">'figures/exp3/init_pred_hist.jpg'</span><span class="hl std">,</span> <span class="hl kwc">height</span><span class="hl std">=</span><span class="hl num">4</span><span class="hl std">,</span> <span class="hl kwc">width</span><span class="hl std">=</span><span class="hl num">6</span><span class="hl std">)</span>
</pre></div>
</div><div class="rimage center"><img src="figure/analyze-experiment-3-Rhtmlauto-report-3.png" title="plot of chunk auto-report" alt="plot of chunk auto-report" class="plot" /></div><div class="rcode">
<div class="source"><pre class="knitr r"><span class="hl kwd">nrow</span><span class="hl std">(woa_responses[woa_responses</span><span class="hl opt">$</span><span class="hl std">user_init_pred</span> <span class="hl opt">==</span> <span class="hl num">0</span><span class="hl std">,])</span>
</pre></div>
<div class="output"><pre class="knitr r">## [1] 0
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">nrow</span><span class="hl std">(woa_responses[woa_responses</span><span class="hl opt">$</span><span class="hl std">user_init_pred</span> <span class="hl opt">==</span> <span class="hl num">3</span><span class="hl std">,])</span>
</pre></div>
<div class="output"><pre class="knitr r">## [1] 45
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl kwd">nrow</span><span class="hl std">(woa_responses[woa_responses</span><span class="hl opt">$</span><span class="hl std">user_init_pred</span> <span class="hl opt">==</span> <span class="hl num">1.5</span><span class="hl std">,])</span>
</pre></div>
<div class="output"><pre class="knitr r">## [1] 626
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com">##########</span>
<span class="hl com"># limit everything to first 10 questions for now</span>
<span class="hl std">woa_responses_normal</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(woa_responses, q_id</span> <span class="hl opt"><</span> <span class="hl num">10</span><span class="hl std">)</span>
<span class="hl std">model_expert_data</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(woa_responses_normal, q_id</span> <span class="hl opt"><</span> <span class="hl num">10</span><span class="hl std">)</span><span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">transparency</span> <span class="hl std">=</span> <span class="hl kwd">ifelse</span><span class="hl std">(condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-2"</span> <span class="hl opt">|</span> <span class="hl std">condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-8"</span><span class="hl std">,</span> <span class="hl str">"CLEAR"</span><span class="hl std">,</span> <span class="hl str">"BB"</span><span class="hl std">))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">num_features</span> <span class="hl std">=</span> <span class="hl kwd">ifelse</span><span class="hl std">(condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-2"</span> <span class="hl opt">|</span> <span class="hl std">condition</span> <span class="hl opt">==</span> <span class="hl str">"BB-2"</span><span class="hl std">,</span> <span class="hl str">"two"</span><span class="hl std">,</span> <span class="hl str">"eight"</span><span class="hl std">))</span>
<span class="hl com">########################################</span>
<span class="hl com"># prediction error</span>
<span class="hl com">########################################</span>
<span class="hl std">model_data</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(model_expert_data, condition</span> <span class="hl opt">!=</span> <span class="hl str">"EXPERT"</span><span class="hl std">)</span>
<span class="hl std">hlm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lmer</span><span class="hl std">(pred_err</span> <span class="hl opt">~</span> <span class="hl std">transparency</span><span class="hl opt">*</span><span class="hl std">num_features</span> <span class="hl opt">+</span> <span class="hl std">(</span><span class="hl num">1</span><span class="hl opt">|</span><span class="hl std">worker_id),</span> <span class="hl kwc">data</span><span class="hl std">=model_data)</span>
<span class="hl kwd">summary</span><span class="hl std">(hlm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
## Formula: pred_err ~ transparency * num_features + (1 | worker_id)
## Data: model_data
##
## REML criterion at convergence: -4522.6
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.0450 -0.6638 -0.1195 0.4750 9.2910
##
## Random effects:
## Groups Name Variance Std.Dev.
## worker_id (Intercept) 0.001278 0.03574
## Residual 0.032089 0.17913
## Number of obs: 8020, groups: worker_id, 802
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 0.217980 0.004760 798.000013 45.793 <2e-16 ***
## transparencyCLEAR 0.002870 0.006715 798.000010 0.427 0.669
## num_featurestwo 0.008456 0.006698 798.000010 1.262 0.207
## transparencyCLEAR:num_featurestwo -0.007771 0.009461 798.000008 -0.821 0.412
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) trCLEAR nm_ftr
## trnsprCLEAR -0.709
## num_fetrstw -0.711 0.504
## trnsCLEAR:_ 0.503 -0.710 -0.708
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># get the marginal means for all combinations of transparency and number of features</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">emmeans</span><span class="hl std">(hlm_model,</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl str">"transparency"</span><span class="hl std">,</span> <span class="hl str">"num_features"</span><span class="hl std">))</span>
<span class="hl com"># setup contrasts</span>
<span class="hl com"># clear vs. bb</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">clear_vs_bb</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## clear_vs_bb -0.00203 0.00946 798 -0.215 0.8301
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com">########################################</span>
<span class="hl com"># deviation</span>
<span class="hl com">########################################</span>
<span class="hl std">model_data</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(model_expert_data, condition</span> <span class="hl opt">!=</span> <span class="hl str">"EXPERT"</span><span class="hl std">)</span>
<span class="hl com"># clear2 vs bb8</span>
<span class="hl std">hlm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lmer</span><span class="hl std">(deviation</span> <span class="hl opt">~</span> <span class="hl std">transparency</span><span class="hl opt">*</span><span class="hl std">num_features</span> <span class="hl opt">+</span> <span class="hl std">(</span><span class="hl num">1</span><span class="hl opt">|</span><span class="hl std">worker_id),</span> <span class="hl kwc">data</span><span class="hl std">=model_data)</span>
<span class="hl kwd">summary</span><span class="hl std">(hlm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
## Formula: deviation ~ transparency * num_features + (1 | worker_id)
## Data: model_data
##
## REML criterion at convergence: -9160.7
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -4.0573 -0.6125 -0.1010 0.3173 9.4149
##
## Random effects:
## Groups Name Variance Std.Dev.
## worker_id (Intercept) 0.006002 0.07747
## Residual 0.015923 0.12619
## Number of obs: 8020, groups: worker_id, 802
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 0.130707 0.006193 797.999185 21.105 <2e-16 ***
## transparencyCLEAR -0.005857 0.008736 797.999199 -0.670 0.503
## num_featurestwo 0.011768 0.008715 797.999199 1.350 0.177
## transparencyCLEAR:num_featurestwo -0.013499 0.012309 797.999204 -1.097 0.273
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) trCLEAR nm_ftr
## trnsprCLEAR -0.709
## num_fetrstw -0.711 0.504
## trnsCLEAR:_ 0.503 -0.710 -0.708
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at ANOVA</span>
<span class="hl std">anova_table</span> <span class="hl kwb"><-</span> <span class="hl kwd">anova</span><span class="hl std">(hlm_model)</span>
<span class="hl com"># export in LaTEX format</span>
<span class="hl kwd">xtable</span><span class="hl std">(anova_table)</span>
</pre></div>
<div class="output"><pre class="knitr r">## % latex table generated in R 3.5.1 by xtable 1.8-2 package
## % Fri Jan 8 17:58:49 2021
## \begin{table}[ht]
## \centering
## \begin{tabular}{lrrrrrr}
## \hline
## & Sum Sq & Mean Sq & NumDF & DenDF & F value & Pr($>$F) \\
## \hline
## transparency & 0.07 & 0.07 & 1.00 & 798.00 & 4.20 & 0.0409 \\
## num\_features & 0.01 & 0.01 & 1.00 & 798.00 & 0.66 & 0.4151 \\
## transparency:num\_features & 0.02 & 0.02 & 1.00 & 798.00 & 1.20 & 0.2731 \\
## \hline
## \end{tabular}
## \end{table}
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># get the marginal means for all combinations of transparency and number of features</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">emmeans</span><span class="hl std">(hlm_model,</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl str">"transparency"</span><span class="hl std">,</span> <span class="hl str">"num_features"</span><span class="hl std">))</span>
<span class="hl com"># setup contrasts</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">clear2_vs_bb8</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## clear2_vs_bb8 -0.00759 0.00871 798 -0.871 0.3842
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># POST-HOC: clear-2 else</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">clear2_vs_else</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">3</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## clear2_vs_else -0.0287 0.0213 798 -1.348 0.1779
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># bb8 vs EXPERT</span>
<span class="hl std">hlm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lmer</span><span class="hl std">(deviation</span> <span class="hl opt">~</span> <span class="hl std">condition</span> <span class="hl opt">+</span> <span class="hl std">(</span><span class="hl num">1</span><span class="hl opt">|</span><span class="hl std">worker_id),</span> <span class="hl kwc">data</span><span class="hl std">=model_expert_data)</span>
<span class="hl kwd">summary</span><span class="hl std">(hlm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
## Formula: deviation ~ condition + (1 | worker_id)
## Data: model_expert_data
##
## REML criterion at convergence: -11377.8
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -5.4543 -0.6006 -0.1045 0.3039 9.3717
##
## Random effects:
## Groups Name Variance Std.Dev.
## worker_id (Intercept) 0.006767 0.08226
## Residual 0.015815 0.12576
## Number of obs: 9990, groups: worker_id, 999
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 1.231e-01 6.429e-03 9.940e+02 19.151 <2e-16 ***
## conditionCLEAR-8 1.731e-03 9.115e-03 9.940e+02 0.190 0.8494
## conditionBB-2 1.936e-02 9.092e-03 9.940e+02 2.129 0.0335 *
## conditionBB-8 7.588e-03 9.138e-03 9.940e+02 0.830 0.4065
## conditionEXPERT 3.480e-03 9.149e-03 9.940e+02 0.380 0.7038
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) cCLEAR cnBB-2 cnBB-8
## cndtCLEAR-8 -0.705
## conditnBB-2 -0.707 0.499
## conditnBB-8 -0.704 0.496 0.497
## cndtnEXPERT -0.703 0.496 0.497 0.494
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at ANOVA</span>
<span class="hl std">anova_table</span> <span class="hl kwb"><-</span> <span class="hl kwd">anova</span><span class="hl std">(hlm_model)</span>
<span class="hl com"># export in LaTEX format</span>
<span class="hl kwd">xtable</span><span class="hl std">(anova_table)</span>
</pre></div>
<div class="output"><pre class="knitr r">## % latex table generated in R 3.5.1 by xtable 1.8-2 package
## % Fri Jan 8 17:58:49 2021
## \begin{table}[ht]
## \centering
## \begin{tabular}{lrrrrrr}
## \hline
## & Sum Sq & Mean Sq & NumDF & DenDF & F value & Pr($>$F) \\
## \hline
## condition & 0.09 & 0.02 & 4.00 & 994.00 & 1.45 & 0.2148 \\
## \hline
## \end{tabular}
## \end{table}
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># get the marginal means for all combinations of transparency and number of features</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">emmeans</span><span class="hl std">(hlm_model,</span> <span class="hl str">"condition"</span><span class="hl std">)</span>
<span class="hl com"># setup contrasts</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">bb8_vs_expert</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## bb8_vs_expert 0.00411 0.00919 994 0.447 0.6551
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># compute the means and standard errors by condition from the model</span>
<span class="hl std">hlm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lmer</span><span class="hl std">(deviation</span> <span class="hl opt">~</span> <span class="hl std">condition</span> <span class="hl opt">+</span> <span class="hl std">(</span><span class="hl num">1</span> <span class="hl opt">|</span> <span class="hl std">worker_id),</span> <span class="hl kwc">data</span> <span class="hl std">= model_expert_data)</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">lsmeansLT</span><span class="hl std">(hlm_model,</span> <span class="hl str">"condition"</span><span class="hl std">)</span>
<span class="hl std">model_expert_data</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">outcome</span> <span class="hl std">= deviation</span> <span class="hl opt">*</span> <span class="hl num">1e3</span><span class="hl std">)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">plot_distributions_with_means</span><span class="hl std">(.,</span> <span class="hl str">'Average deviation from the model'</span><span class="hl std">,</span> <span class="hl str">'Proportion of participants'</span><span class="hl std">, my_pallete)</span>
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: Ignoring unknown aesthetics: x
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: The plyr::rename operation has created duplicates for the following name(s):
## (`size`)
</pre></div>
<div class="message"><pre class="knitr r">## Picking joint bandwidth of 16.5
</pre></div>
</div><div class="rimage center"><img src="figure/analyze-experiment-3-Rhtmlauto-report-4.png" title="plot of chunk auto-report" alt="plot of chunk auto-report" class="plot" /></div><div class="rcode">
<div class="source"><pre class="knitr r"><span class="hl kwd">ggsave</span><span class="hl std">(</span><span class="hl kwc">file</span> <span class="hl std">=</span> <span class="hl str">'figures/exp3/dev_from_model.pdf'</span><span class="hl std">,</span> <span class="hl kwc">height</span> <span class="hl std">=</span> <span class="hl num">4.5</span><span class="hl std">,</span> <span class="hl kwc">width</span> <span class="hl std">=</span> <span class="hl num">4</span><span class="hl std">)</span>
</pre></div>
<div class="message"><pre class="knitr r">## Picking joint bandwidth of 16.5
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com">########################################</span>
<span class="hl com"># WEIGHT OF ADVICE</span>
<span class="hl com">########################################</span>
<span class="hl com"># keeping only finite (non Inf/NA) woa values</span>
<span class="hl std">model_expert_data_woa</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(model_expert_data,</span> <span class="hl kwd">is.finite</span><span class="hl std">(woa))</span>
<span class="hl std">model_data_woa</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(model_expert_data_woa, condition</span> <span class="hl opt">!=</span> <span class="hl str">"EXPERT"</span><span class="hl std">)</span>
<span class="hl com"># clear2 vs bb8</span>
<span class="hl std">hlm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lmer</span><span class="hl std">(woa</span> <span class="hl opt">~</span> <span class="hl std">transparency</span><span class="hl opt">*</span><span class="hl std">num_features</span> <span class="hl opt">+</span> <span class="hl std">(</span><span class="hl num">1</span><span class="hl opt">|</span><span class="hl std">worker_id),</span> <span class="hl kwc">data</span><span class="hl std">=model_data_woa)</span>
<span class="hl kwd">summary</span><span class="hl std">(hlm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
## Formula: woa ~ transparency * num_features + (1 | worker_id)
## Data: model_data_woa
##
## REML criterion at convergence: 9594.8
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -3.9891 -0.5647 -0.0357 0.4674 10.8070
##
## Random effects:
## Groups Name Variance Std.Dev.
## worker_id (Intercept) 0.05041 0.2245
## Residual 0.20449 0.4522
## Number of obs: 6927, groups: worker_id, 802
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 0.60350 0.01932 807.84408 31.235 <2e-16 ***
## transparencyCLEAR 0.03822 0.02728 809.88236 1.401 0.1616
## num_featurestwo -0.05181 0.02725 814.60662 -1.901 0.0576 .
## transparencyCLEAR:num_featurestwo 0.04823 0.03853 817.77441 1.252 0.2109
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) trCLEAR nm_ftr
## trnsprCLEAR -0.708
## num_fetrstw -0.709 0.502
## trnsCLEAR:_ 0.502 -0.708 -0.707
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at ANOVA</span>
<span class="hl std">anova_table</span> <span class="hl kwb"><-</span> <span class="hl kwd">anova</span><span class="hl std">(hlm_model)</span>
<span class="hl com"># export in LaTEX format</span>
<span class="hl kwd">xtable</span><span class="hl std">(anova_table)</span>
</pre></div>
<div class="output"><pre class="knitr r">## % latex table generated in R 3.5.1 by xtable 1.8-2 package
## % Fri Jan 8 17:58:50 2021
## \begin{table}[ht]
## \centering
## \begin{tabular}{lrrrrrr}
## \hline
## & Sum Sq & Mean Sq & NumDF & DenDF & F value & Pr($>$F) \\
## \hline
## transparency & 2.14 & 2.14 & 1.00 & 817.77 & 10.47 & 0.0013 \\
## num\_features & 0.42 & 0.42 & 1.00 & 817.77 & 2.07 & 0.1509 \\
## transparency:num\_features & 0.32 & 0.32 & 1.00 & 817.77 & 1.57 & 0.2109 \\
## \hline
## \end{tabular}
## \end{table}
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># get the marginal means for all combinations of transparency and number of features</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">emmeans</span><span class="hl std">(hlm_model,</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl str">"transparency"</span><span class="hl std">,</span> <span class="hl str">"num_features"</span><span class="hl std">))</span>
<span class="hl com"># setup contrasts</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">clear2_vs_bb8</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## clear2_vs_bb8 0.0346 0.0273 819 1.270 0.2045
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># clear-2 else</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">clear2_vs_else</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">3</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## clear2_vs_else 0.118 0.0667 826 1.761 0.0786
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># bb8 vs EXPERT</span>
<span class="hl std">hlm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lmer</span><span class="hl std">(woa</span> <span class="hl opt">~</span> <span class="hl std">condition</span> <span class="hl opt">+</span> <span class="hl std">(</span><span class="hl num">1</span><span class="hl opt">|</span><span class="hl std">worker_id),</span> <span class="hl kwc">data</span><span class="hl std">=model_expert_data_woa)</span>
<span class="hl kwd">summary</span><span class="hl std">(hlm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
## Formula: woa ~ condition + (1 | worker_id)
## Data: model_expert_data_woa
##
## REML criterion at convergence: 11517
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -4.0890 -0.5558 -0.0275 0.4762 11.0996
##
## Random effects:
## Groups Name Variance Std.Dev.
## worker_id (Intercept) 0.04742 0.2178
## Residual 0.19395 0.4404
## Number of obs: 8650, groups: worker_id, 999
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 6.381e-01 1.871e-02 1.031e+03 34.113 <2e-16 ***
## conditionCLEAR-8 3.575e-03 2.645e-02 1.019e+03 0.135 0.8925
## conditionBB-2 -8.646e-02 2.642e-02 1.025e+03 -3.272 0.0011 **
## conditionBB-8 -3.465e-02 2.650e-02 1.017e+03 -1.308 0.1913
## conditionEXPERT -2.458e-02 2.655e-02 1.019e+03 -0.926 0.3548
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) cCLEAR cnBB-2 cnBB-8
## cndtCLEAR-8 -0.707
## conditnBB-2 -0.708 0.501
## conditnBB-8 -0.706 0.499 0.500
## cndtnEXPERT -0.705 0.498 0.499 0.498
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at ANOVA</span>
<span class="hl std">anova_table</span> <span class="hl kwb"><-</span> <span class="hl kwd">anova</span><span class="hl std">(hlm_model)</span>
<span class="hl com"># export in LaTEX format</span>
<span class="hl kwd">xtable</span><span class="hl std">(anova_table)</span>
</pre></div>
<div class="output"><pre class="knitr r">## % latex table generated in R 3.5.1 by xtable 1.8-2 package
## % Fri Jan 8 17:58:50 2021
## \begin{table}[ht]
## \centering
## \begin{tabular}{lrrrrrr}
## \hline
## & Sum Sq & Mean Sq & NumDF & DenDF & F value & Pr($>$F) \\
## \hline
## condition & 2.92 & 0.73 & 4.00 & 1013.65 & 3.77 & 0.0048 \\
## \hline
## \end{tabular}
## \end{table}
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># get the marginal means for all combinations of transparency and number of features</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">emmeans</span><span class="hl std">(hlm_model,</span> <span class="hl str">"condition"</span><span class="hl std">)</span>
<span class="hl com"># setup contrasts</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">bb8_vs_expert</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">0</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## bb8_vs_expert -0.0101 0.0266 1005 -0.379 0.7049
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl std">hlm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lmer</span><span class="hl std">(woa</span> <span class="hl opt">~</span> <span class="hl std">condition</span> <span class="hl opt">+</span> <span class="hl std">(</span><span class="hl num">1</span> <span class="hl opt">|</span> <span class="hl std">worker_id),</span> <span class="hl kwc">data</span> <span class="hl std">= model_expert_data_woa)</span>
<span class="hl kwd">summary</span><span class="hl std">(hlm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
## Formula: woa ~ condition + (1 | worker_id)
## Data: model_expert_data_woa
##
## REML criterion at convergence: 11517
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -4.0890 -0.5558 -0.0275 0.4762 11.0996
##
## Random effects:
## Groups Name Variance Std.Dev.
## worker_id (Intercept) 0.04742 0.2178
## Residual 0.19395 0.4404
## Number of obs: 8650, groups: worker_id, 999
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 6.381e-01 1.871e-02 1.031e+03 34.113 <2e-16 ***
## conditionCLEAR-8 3.575e-03 2.645e-02 1.019e+03 0.135 0.8925
## conditionBB-2 -8.646e-02 2.642e-02 1.025e+03 -3.272 0.0011 **
## conditionBB-8 -3.465e-02 2.650e-02 1.017e+03 -1.308 0.1913
## conditionEXPERT -2.458e-02 2.655e-02 1.019e+03 -0.926 0.3548
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) cCLEAR cnBB-2 cnBB-8
## cndtCLEAR-8 -0.707
## conditnBB-2 -0.708 0.501
## conditnBB-8 -0.706 0.499 0.500
## cndtnEXPERT -0.705 0.498 0.499 0.498
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl std">model_expert_data_woa</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">outcome</span> <span class="hl std">= woa)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">plot_distributions_with_means</span><span class="hl std">(.,</span> <span class="hl str">'Average weight of advice (WOA)'</span><span class="hl std">,</span> <span class="hl str">'Proportion of participants'</span><span class="hl std">, my_pallete,</span> <span class="hl kwc">label_format</span> <span class="hl std">=</span> <span class="hl str">'%.2f'</span><span class="hl std">)</span>
</pre></div>
<div class="warning"><pre class="knitr r">## Warning: Ignoring unknown aesthetics: x
## Warning: The plyr::rename operation has created duplicates for the following name(s):
## (`size`)
</pre></div>
<div class="message"><pre class="knitr r">## Picking joint bandwidth of 0.0698
</pre></div>
</div><div class="rimage center"><img src="figure/analyze-experiment-3-Rhtmlauto-report-5.png" title="plot of chunk auto-report" alt="plot of chunk auto-report" class="plot" /></div><div class="rcode">
<div class="source"><pre class="knitr r"><span class="hl kwd">ggsave</span><span class="hl std">(</span><span class="hl kwc">file</span> <span class="hl std">=</span> <span class="hl str">'figures/exp3/woa.pdf'</span><span class="hl std">,</span> <span class="hl kwc">height</span> <span class="hl std">=</span> <span class="hl num">4.5</span><span class="hl std">,</span> <span class="hl kwc">width</span> <span class="hl std">=</span> <span class="hl num">4</span><span class="hl std">)</span>
</pre></div>
<div class="message"><pre class="knitr r">## Picking joint bandwidth of 0.0698
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># response to R2</span>
<span class="hl com"># how many WOA responses are outside of [0,1]?</span>
<span class="hl com"># overall</span>
<span class="hl std">model_data</span> <span class="hl opt">%>%</span>
<span class="hl kwd">filter</span><span class="hl std">(q_id</span> <span class="hl opt"><</span> <span class="hl num">10</span><span class="hl std">)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">woa_bin</span> <span class="hl std">=</span> <span class="hl kwd">case_when</span><span class="hl std">(</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(user_init_pred</span> <span class="hl opt"><=</span> <span class="hl std">final_pred</span> <span class="hl opt">&</span> <span class="hl std">final_pred</span> <span class="hl opt"><=</span> <span class="hl std">model_pred)</span><span class="hl opt">~</span> <span class="hl str">"u1 < u2 < m (woa <= 1)"</span><span class="hl std">,</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(final_pred</span> <span class="hl opt"><=</span> <span class="hl std">user_init_pred</span> <span class="hl opt">&</span> <span class="hl std">user_init_pred</span> <span class="hl opt"><=</span> <span class="hl std">model_pred)</span> <span class="hl opt">&</span> <span class="hl std">woa</span> <span class="hl opt"><=</span> <span class="hl num">1</span> <span class="hl opt">~</span> <span class="hl str">"u2 < u1 < m and woa <= 1"</span><span class="hl std">,</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(final_pred</span> <span class="hl opt"><=</span> <span class="hl std">user_init_pred</span> <span class="hl opt">&</span> <span class="hl std">user_init_pred</span> <span class="hl opt"><=</span> <span class="hl std">model_pred)</span> <span class="hl opt">&</span> <span class="hl std">woa</span> <span class="hl opt">></span> <span class="hl num">1</span> <span class="hl opt">~</span> <span class="hl str">"u2 < u1 < m and woa > 1"</span><span class="hl std">,</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(user_init_pred</span> <span class="hl opt"><=</span> <span class="hl std">model_pred</span> <span class="hl opt">&</span> <span class="hl std">model_pred</span> <span class="hl opt"><=</span> <span class="hl std">final_pred)</span><span class="hl opt">~</span> <span class="hl str">"u1 < m < u2 (woa > 1)"</span><span class="hl std">,</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(model_pred</span> <span class="hl opt"><=</span> <span class="hl std">final_pred</span> <span class="hl opt">&</span> <span class="hl std">final_pred</span> <span class="hl opt"><=</span> <span class="hl std">user_init_pred)</span><span class="hl opt">~</span> <span class="hl str">" m < u2 < u1 (woa <= 1)"</span><span class="hl std">,</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(final_pred</span> <span class="hl opt"><=</span> <span class="hl std">model_pred</span> <span class="hl opt">&</span> <span class="hl std">model_pred</span> <span class="hl opt"><=</span> <span class="hl std">user_init_pred)</span><span class="hl opt">~</span> <span class="hl str">" u2 < m < u1 (woa > 1)"</span><span class="hl std">,</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(model_pred</span> <span class="hl opt"><=</span> <span class="hl std">user_init_pred</span> <span class="hl opt">&</span> <span class="hl std">user_init_pred</span> <span class="hl opt"><=</span> <span class="hl std">final_pred)</span> <span class="hl opt">&</span> <span class="hl std">woa</span> <span class="hl opt"><=</span> <span class="hl num">1</span> <span class="hl opt">~</span> <span class="hl str">"m < u1 < u2 and woa <= 1"</span><span class="hl std">,</span>
<span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">&</span> <span class="hl std">(model_pred</span> <span class="hl opt"><=</span> <span class="hl std">user_init_pred</span> <span class="hl opt">&</span> <span class="hl std">user_init_pred</span> <span class="hl opt"><=</span> <span class="hl std">final_pred)</span> <span class="hl opt">&</span> <span class="hl std">woa</span> <span class="hl opt">></span> <span class="hl num">1</span> <span class="hl opt">~</span> <span class="hl str">"m < u1 < u2 and woa > 1"</span><span class="hl std">,</span>
<span class="hl opt">!</span><span class="hl kwd">is.finite</span><span class="hl std">(woa)</span> <span class="hl opt">~</span> <span class="hl str">"WOA undefined\n(Initial prediction matched model)"</span><span class="hl std">,</span>
<span class="hl num">TRUE</span> <span class="hl opt">~</span> <span class="hl str">"Other"</span>
<span class="hl std">))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">count</span><span class="hl std">(woa_bin)</span> <span class="hl opt">%>%</span>
<span class="hl kwd">ungroup</span><span class="hl std">()</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">frac_in_bin</span> <span class="hl std">= n</span> <span class="hl opt">/</span> <span class="hl kwd">sum</span><span class="hl std">(n),</span>
<span class="hl kwc">se</span> <span class="hl std">=</span> <span class="hl kwd">sqrt</span><span class="hl std">(frac_in_bin</span> <span class="hl opt">*</span> <span class="hl std">(</span><span class="hl num">1</span> <span class="hl opt">-</span> <span class="hl std">frac_in_bin)</span> <span class="hl opt">/</span> <span class="hl std">n))</span>
</pre></div>
<div class="output"><pre class="knitr r">## # A tibble: 9 x 4
## woa_bin n frac_in_bin se
## <chr> <int> <dbl> <dbl>
## 1 " m < u2 < u1 (woa <= 1)" 3820 0.476 0.00808
## 2 " u2 < m < u1 (woa > 1)" 194 0.0242 0.0110
## 3 "m < u1 < u2 and woa <= 1" 49 0.00611 0.0111
## 4 "m < u1 < u2 and woa > 1" 33 0.00411 0.0111
## 5 "u1 < m < u2 (woa > 1)" 224 0.0279 0.0110
## 6 "u1 < u2 < m (woa <= 1)" 2571 0.321 0.00920
## 7 "u2 < u1 < m and woa <= 1" 22 0.00274 0.0112
## 8 "u2 < u1 < m and woa > 1" 14 0.00175 0.0112
## 9 "WOA undefined\n(Initial prediction matched model)" 1093 0.136 0.0104
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com">####################################</span>
<span class="hl com"># last two questions (unusual apartments)</span>
<span class="hl com">####################################</span>
<span class="hl std">woa_responses_unusual</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(woa_responses, q_id</span> <span class="hl opt">>=</span> <span class="hl num">10</span><span class="hl std">)</span>
<span class="hl com">########################################</span>
<span class="hl com"># deviation for q11</span>
<span class="hl com">########################################</span>
<span class="hl com"># anova for q11</span>
<span class="hl std">q11_model_data</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(woa_responses_unusual, condition</span> <span class="hl opt">!=</span> <span class="hl str">"EXPERT"</span><span class="hl std">, q_id</span> <span class="hl opt">==</span> <span class="hl num">10</span><span class="hl std">)</span><span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">transparency</span> <span class="hl std">=</span> <span class="hl kwd">ifelse</span><span class="hl std">(condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-2"</span> <span class="hl opt">|</span> <span class="hl std">condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-8"</span><span class="hl std">,</span> <span class="hl str">"CLEAR"</span><span class="hl std">,</span> <span class="hl str">"BB"</span><span class="hl std">))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">num_features</span> <span class="hl std">=</span> <span class="hl kwd">ifelse</span><span class="hl std">(condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-2"</span> <span class="hl opt">|</span> <span class="hl std">condition</span> <span class="hl opt">==</span> <span class="hl str">"BB-2"</span><span class="hl std">,</span> <span class="hl str">"two"</span><span class="hl std">,</span> <span class="hl str">"eight"</span><span class="hl std">))</span>
<span class="hl com"># fit the one factor model</span>
<span class="hl std">lm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lm</span><span class="hl std">(deviation</span> <span class="hl opt">~</span> <span class="hl std">condition,</span> <span class="hl kwc">data</span><span class="hl std">=q11_model_data)</span>
<span class="hl kwd">summary</span><span class="hl std">(lm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">##
## Call:
## lm(formula = deviation ~ condition, data = q11_model_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -0.14901 -0.04901 -0.02980 0.05495 0.67020
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 0.149010 0.008208 18.155 < 2e-16 ***
## conditionCLEAR-8 -0.039010 0.011636 -3.352 0.000839 ***
## conditionBB-2 -0.003960 0.011607 -0.341 0.733045
## conditionBB-8 -0.019212 0.011666 -1.647 0.099982 .
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 0.1167 on 798 degrees of freedom
## Multiple R-squared: 0.01708, Adjusted R-squared: 0.01339
## F-statistic: 4.622 on 3 and 798 DF, p-value: 0.003256
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at one-way ANOVA</span>
<span class="hl kwd">anova</span><span class="hl std">(lm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Analysis of Variance Table
##
## Response: deviation
## Df Sum Sq Mean Sq F value Pr(>F)
## condition 3 0.1887 0.062900 4.6223 0.003256 **
## Residuals 798 10.8590 0.013608
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># run 2-by-2 anova for deviation on q11</span>
<span class="hl com"># fit the two factor model</span>
<span class="hl std">lm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lm</span><span class="hl std">(deviation</span> <span class="hl opt">~</span> <span class="hl std">transparency</span><span class="hl opt">*</span><span class="hl std">num_features,</span> <span class="hl kwc">data</span><span class="hl std">=q11_model_data)</span>
<span class="hl kwd">summary</span><span class="hl std">(lm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">##
## Call:
## lm(formula = deviation ~ transparency * num_features, data = q11_model_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -0.14901 -0.04901 -0.02980 0.05495 0.67020
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 0.12980 0.00829 15.657 <2e-16 ***
## transparencyCLEAR -0.01980 0.01170 -1.693 0.0909 .
## num_featurestwo 0.01525 0.01167 1.307 0.1915
## transparencyCLEAR:num_featurestwo 0.02376 0.01648 1.442 0.1497
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 0.1167 on 798 degrees of freedom
## Multiple R-squared: 0.01708, Adjusted R-squared: 0.01339
## F-statistic: 4.622 on 3 and 798 DF, p-value: 0.003256
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at ANOVA</span>
<span class="hl std">anova_table</span> <span class="hl kwb"><-</span> <span class="hl kwd">anova</span><span class="hl std">(lm_model)</span>
<span class="hl com"># export in LaTEX format</span>
<span class="hl kwd">xtable</span><span class="hl std">(anova_table)</span>
</pre></div>
<div class="output"><pre class="knitr r">## % latex table generated in R 3.5.1 by xtable 1.8-2 package
## % Fri Jan 8 17:58:51 2021
## \begin{table}[ht]
## \centering
## \begin{tabular}{lrrrrr}
## \hline
## & Df & Sum Sq & Mean Sq & F value & Pr($>$F) \\
## \hline
## transparency & 1 & 0.01 & 0.01 & 0.92 & 0.3380 \\
## num\_features & 1 & 0.15 & 0.15 & 10.87 & 0.0010 \\
## transparency:num\_features & 1 & 0.03 & 0.03 & 2.08 & 0.1497 \\
## Residuals & 798 & 10.86 & 0.01 & & \\
## \hline
## \end{tabular}
## \end{table}
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># get the marginal means for all combinations of transparency and number of features</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">emmeans</span><span class="hl std">(lm_model,</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl str">"transparency"</span><span class="hl std">,</span> <span class="hl str">"num_features"</span><span class="hl std">))</span>
<span class="hl com"># setup contrasts</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">clear_vs_bb</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## clear_vs_bb -0.0158 0.0165 798 -0.961 0.3368
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com">########################################</span>
<span class="hl com"># DEVIATION for q12</span>
<span class="hl com">########################################</span>
<span class="hl com"># anova for q12</span>
<span class="hl std">q12_model_data</span> <span class="hl kwb"><-</span> <span class="hl kwd">filter</span><span class="hl std">(woa_responses_unusual, condition</span> <span class="hl opt">!=</span> <span class="hl str">"EXPERT"</span><span class="hl std">, q_id</span> <span class="hl opt">==</span> <span class="hl num">11</span><span class="hl std">)</span><span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">transparency</span> <span class="hl std">=</span> <span class="hl kwd">ifelse</span><span class="hl std">(condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-2"</span> <span class="hl opt">|</span> <span class="hl std">condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-8"</span><span class="hl std">,</span> <span class="hl str">"CLEAR"</span><span class="hl std">,</span> <span class="hl str">"BB"</span><span class="hl std">))</span> <span class="hl opt">%>%</span>
<span class="hl kwd">mutate</span><span class="hl std">(</span><span class="hl kwc">num_features</span> <span class="hl std">=</span> <span class="hl kwd">ifelse</span><span class="hl std">(condition</span> <span class="hl opt">==</span> <span class="hl str">"CLEAR-2"</span> <span class="hl opt">|</span> <span class="hl std">condition</span> <span class="hl opt">==</span> <span class="hl str">"BB-2"</span><span class="hl std">,</span> <span class="hl str">"two"</span><span class="hl std">,</span> <span class="hl str">"eight"</span><span class="hl std">))</span>
<span class="hl com"># fit the one factor model</span>
<span class="hl std">lm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lm</span><span class="hl std">(deviation</span> <span class="hl opt">~</span> <span class="hl std">condition,</span> <span class="hl kwc">data</span><span class="hl std">=q12_model_data)</span>
<span class="hl kwd">summary</span><span class="hl std">(lm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">##
## Call:
## lm(formula = deviation ~ condition, data = q12_model_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -0.25891 -0.15347 -0.03582 0.09250 0.94653
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 0.258911 0.013488 19.196 < 2e-16 ***
## conditionCLEAR-8 -0.051411 0.019122 -2.689 0.00733 **
## conditionBB-2 -0.005446 0.019075 -0.285 0.77535
## conditionBB-8 -0.040729 0.019171 -2.125 0.03393 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 0.1917 on 798 degrees of freedom
## Multiple R-squared: 0.01318, Adjusted R-squared: 0.009466
## F-statistic: 3.551 on 3 and 798 DF, p-value: 0.01416
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at one-way ANOVA</span>
<span class="hl kwd">anova</span><span class="hl std">(lm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">## Analysis of Variance Table
##
## Response: deviation
## Df Sum Sq Mean Sq F value Pr(>F)
## condition 3 0.3915 0.130510 3.5515 0.01416 *
## Residuals 798 29.3248 0.036748
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># run 2-by-2 anova for deviation on q11</span>
<span class="hl com"># fit the two factor model</span>
<span class="hl std">lm_model</span> <span class="hl kwb"><-</span> <span class="hl kwd">lm</span><span class="hl std">(deviation</span> <span class="hl opt">~</span> <span class="hl std">transparency</span><span class="hl opt">*</span><span class="hl std">num_features,</span> <span class="hl kwc">data</span><span class="hl std">=q12_model_data)</span>
<span class="hl kwd">summary</span><span class="hl std">(lm_model)</span>
</pre></div>
<div class="output"><pre class="knitr r">##
## Call:
## lm(formula = deviation ~ transparency * num_features, data = q12_model_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -0.25891 -0.15347 -0.03582 0.09250 0.94653
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 0.21818 0.01362 16.015 <2e-16 ***
## transparencyCLEAR -0.01068 0.01922 -0.556 0.5785
## num_featurestwo 0.03528 0.01917 1.840 0.0661 .
## transparencyCLEAR:num_featurestwo 0.01613 0.02708 0.596 0.5516
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 0.1917 on 798 degrees of freedom
## Multiple R-squared: 0.01318, Adjusted R-squared: 0.009466
## F-statistic: 3.551 on 3 and 798 DF, p-value: 0.01416
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># look at ANOVA</span>
<span class="hl std">anova_table</span> <span class="hl kwb"><-</span> <span class="hl kwd">anova</span><span class="hl std">(lm_model)</span>
<span class="hl com"># export in LaTEX format</span>
<span class="hl kwd">xtable</span><span class="hl std">(anova_table)</span>
</pre></div>
<div class="output"><pre class="knitr r">## % latex table generated in R 3.5.1 by xtable 1.8-2 package
## % Fri Jan 8 17:58:51 2021
## \begin{table}[ht]
## \centering
## \begin{tabular}{lrrrrr}
## \hline
## & Df & Sum Sq & Mean Sq & F value & Pr($>$F) \\
## \hline
## transparency & 1 & 0.00 & 0.00 & 0.04 & 0.8439 \\
## num\_features & 1 & 0.38 & 0.38 & 10.26 & 0.0014 \\
## transparency:num\_features & 1 & 0.01 & 0.01 & 0.35 & 0.5516 \\
## Residuals & 798 & 29.32 & 0.04 & & \\
## \hline
## \end{tabular}
## \end{table}
</pre></div>
<div class="source"><pre class="knitr r"><span class="hl com"># get the marginal means for all combinations of transparency and number of features</span>
<span class="hl std">means</span> <span class="hl kwb"><-</span> <span class="hl kwd">emmeans</span><span class="hl std">(lm_model,</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl str">"transparency"</span><span class="hl std">,</span> <span class="hl str">"num_features"</span><span class="hl std">))</span>
<span class="hl com"># setup contrasts</span>
<span class="hl std">clist</span> <span class="hl kwb"><-</span> <span class="hl kwd">list</span><span class="hl std">(</span><span class="hl kwc">clear_vs_bb</span> <span class="hl std">=</span> <span class="hl kwd">c</span><span class="hl std">(</span><span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">,</span> <span class="hl opt">-</span><span class="hl num">1</span><span class="hl std">,</span> <span class="hl num">1</span><span class="hl std">))</span>
<span class="hl std">emmeans</span><span class="hl opt">::</span><span class="hl kwd">contrast</span><span class="hl std">(means, clist)</span>
</pre></div>
<div class="output"><pre class="knitr r">## contrast estimate SE df t.ratio p.value
## clear_vs_bb -0.00524 0.0271 798 -0.193 0.8467
</pre></div>
</div></div>
<p>The R session information (including the OS info, R version and all
packages used):</p>
<div class="chunk" id="session-info"><div class="rcode"><div class="source"><pre class="knitr r"> <span class="hl kwd">sessionInfo</span><span class="hl std">()</span>
</pre></div>
<div class="output"><pre class="knitr r">## R version 3.5.1 (2018-07-02)
## Platform: x86_64-apple-darwin15.6.0 (64-bit)
## Running under: macOS 10.16
##
## Matrix products: default
## BLAS: /Library/Frameworks/R.framework/Versions/3.5/Resources/lib/libRblas.0.dylib
## LAPACK: /Library/Frameworks/R.framework/Versions/3.5/Resources/lib/libRlapack.dylib
##
## locale:
## [1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
##
## attached base packages:
## [1] stats graphics grDevices utils datasets methods base
##
## other attached packages:
## [1] forcats_0.3.0 stringr_1.3.1 purrr_0.2.5 tibble_2.1.3 tidyverse_1.2.1
## [6] ggridges_0.5.1 lsr_0.5 emmeans_1.4 xtable_1.8-2 tidyr_0.8.1
## [11] dplyr_0.8.3 magrittr_1.5 readr_1.1.1 ggrepel_0.8.0 ggplot2_3.0.0
## [16] scales_0.5.0 lsmeans_2.27-62 lmerTest_3.0-1 lme4_1.1-17 Matrix_1.2-14
##
## loaded via a namespace (and not attached):
## [1] Rcpp_1.0.3 lubridate_1.7.4 mvtnorm_1.0-8 lattice_0.20-35
## [5] zoo_1.8-3 zeallot_0.1.0 utf8_1.1.4 digest_0.6.15
## [9] assertthat_0.2.0 R6_2.2.2 cellranger_1.1.0 plyr_1.8.4
## [13] backports_1.1.2 evaluate_0.11 coda_0.19-1 httr_1.3.1
## [17] highr_0.7 pillar_1.4.3 rlang_0.4.2 lazyeval_0.2.1
## [21] multcomp_1.4-8 readxl_1.1.0 minqa_1.2.4 rstudioapi_0.7
## [25] nloptr_1.0.4 labeling_0.3 splines_3.5.1 munsell_0.5.0
## [29] broom_0.5.0 compiler_3.5.1 numDeriv_2016.8-1 modelr_0.1.2
## [33] pkgconfig_2.0.1 tidyselect_0.2.5 codetools_0.2-15 fansi_0.4.1
## [37] crayon_1.3.4 withr_2.1.2 MASS_7.3-50 grid_3.5.1
## [41] nlme_3.1-137 jsonlite_1.5 gtable_0.2.0 estimability_1.3
## [45] cli_1.1.0 stringi_1.2.3 reshape2_1.4.3 xml2_1.2.0
## [49] vctrs_0.2.1 sandwich_2.4-0 TH.data_1.0-9 tools_3.5.1
## [53] glue_1.3.1 hms_0.4.2 survival_2.42-3 colorspace_1.3-2
## [57] rvest_0.3.2 knitr_1.20 haven_1.1.2
</pre></div>
<div class="source"><pre class="knitr r"> <span class="hl kwd">Sys.time</span><span class="hl std">()</span>
</pre></div>
<div class="output"><pre class="knitr r">## [1] "2021-01-08 17:58:53 EST"
</pre></div>
</div></div>
</body>
</html>