From 9998cd7c5eeb913dd7781759a2fcc6e3867ce676 Mon Sep 17 00:00:00 2001 From: Li Gang Date: Tue, 6 Aug 2024 17:38:02 +0800 Subject: [PATCH] Add benchmark stresscli scripts (#50) * Add benchmark stresscli scripts * Profile based multi-run stress tool * Locust based benchmark script for chatQnA and TGI * HW/SW Spec and Metrics collection * Stress result report generation Signed-off-by: Li Gang Co-authored-by: Yingchun Guo Co-authored-by: Cathy Zhang * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refine stresscli as a lib Signed-off-by: lvliang-intel * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: lvliang-intel Co-authored-by: Yingchun Guo Co-authored-by: Cathy Zhang Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: lvliang-intel --- evals/benchmark/stresscli/.gitignore | 7 + evals/benchmark/stresscli/README.md | 132 ++++++++++ evals/benchmark/stresscli/__init__.py | 2 + .../benchmark/stresscli/commands/__init__.py | 2 + evals/benchmark/stresscli/commands/dump.py | 27 ++ .../benchmark/stresscli/commands/load_test.py | 238 ++++++++++++++++++ evals/benchmark/stresscli/commands/main.py | 27 ++ evals/benchmark/stresscli/commands/metrics.py | 122 +++++++++ .../stresscli/commands/metrics_util.py | 204 +++++++++++++++ evals/benchmark/stresscli/commands/report.py | 221 ++++++++++++++++ evals/benchmark/stresscli/commands/utils.py | 198 +++++++++++++++ .../benchmark/stresscli/commands/validate.py | 164 ++++++++++++ evals/benchmark/stresscli/config.ini | 27 ++ .../benchmark/stresscli/dataset/chatqna.json | 191 ++++++++++++++ .../stresscli/dataset/input_llama_3_1.jsonl | 10 + evals/benchmark/stresscli/locust/README.md | 73 ++++++ evals/benchmark/stresscli/locust/__init__.py | 2 + evals/benchmark/stresscli/locust/aistress.py | 157 ++++++++++++ .../stresscli/locust/audioqnabench.py | 38 +++ .../stresscli/locust/audioqnafixed.py | 20 ++ .../stresscli/locust/chatqnabench.py | 38 +++ .../stresscli/locust/chatqnafixed.py | 20 ++ .../stresscli/locust/codegenbench.py | 38 +++ .../stresscli/locust/codegenfixed.py | 20 ++ .../stresscli/locust/codetransbench.py | 38 +++ .../stresscli/locust/codetransfixed.py | 20 ++ .../benchmark/stresscli/locust/faqgenbench.py | 38 +++ .../benchmark/stresscli/locust/faqgenfixed.py | 20 ++ evals/benchmark/stresscli/locust/locust.conf | 9 + evals/benchmark/stresscli/locust/tgifixed.py | 66 +++++ .../stresscli/locust/tokenresponse.py | 99 ++++++++ .../stresscli/locust/visualqnabench.py | 38 +++ .../stresscli/locust/visualqnafixed.py | 20 ++ evals/benchmark/stresscli/requirements.txt | 9 + evals/benchmark/stresscli/run.yaml | 27 ++ evals/benchmark/stresscli/stresscli.py | 9 + 36 files changed, 2371 insertions(+) create mode 100644 evals/benchmark/stresscli/.gitignore create mode 100644 evals/benchmark/stresscli/README.md create mode 100644 evals/benchmark/stresscli/__init__.py create mode 100644 evals/benchmark/stresscli/commands/__init__.py create mode 100644 evals/benchmark/stresscli/commands/dump.py create mode 100644 evals/benchmark/stresscli/commands/load_test.py create mode 100644 evals/benchmark/stresscli/commands/main.py create mode 100644 evals/benchmark/stresscli/commands/metrics.py create mode 100644 evals/benchmark/stresscli/commands/metrics_util.py create mode 100644 evals/benchmark/stresscli/commands/report.py create mode 100644 evals/benchmark/stresscli/commands/utils.py create mode 100644 evals/benchmark/stresscli/commands/validate.py create mode 100644 evals/benchmark/stresscli/config.ini create mode 100644 evals/benchmark/stresscli/dataset/chatqna.json create mode 100644 evals/benchmark/stresscli/dataset/input_llama_3_1.jsonl create mode 100644 evals/benchmark/stresscli/locust/README.md create mode 100644 evals/benchmark/stresscli/locust/__init__.py create mode 100644 evals/benchmark/stresscli/locust/aistress.py create mode 100644 evals/benchmark/stresscli/locust/audioqnabench.py create mode 100644 evals/benchmark/stresscli/locust/audioqnafixed.py create mode 100644 evals/benchmark/stresscli/locust/chatqnabench.py create mode 100644 evals/benchmark/stresscli/locust/chatqnafixed.py create mode 100644 evals/benchmark/stresscli/locust/codegenbench.py create mode 100644 evals/benchmark/stresscli/locust/codegenfixed.py create mode 100644 evals/benchmark/stresscli/locust/codetransbench.py create mode 100644 evals/benchmark/stresscli/locust/codetransfixed.py create mode 100644 evals/benchmark/stresscli/locust/faqgenbench.py create mode 100644 evals/benchmark/stresscli/locust/faqgenfixed.py create mode 100644 evals/benchmark/stresscli/locust/locust.conf create mode 100644 evals/benchmark/stresscli/locust/tgifixed.py create mode 100644 evals/benchmark/stresscli/locust/tokenresponse.py create mode 100644 evals/benchmark/stresscli/locust/visualqnabench.py create mode 100644 evals/benchmark/stresscli/locust/visualqnafixed.py create mode 100644 evals/benchmark/stresscli/requirements.txt create mode 100644 evals/benchmark/stresscli/run.yaml create mode 100755 evals/benchmark/stresscli/stresscli.py diff --git a/evals/benchmark/stresscli/.gitignore b/evals/benchmark/stresscli/.gitignore new file mode 100644 index 00000000..8444a033 --- /dev/null +++ b/evals/benchmark/stresscli/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*testspec.yaml +*exceptions.csv +*failures.csv +*stats_history.csv +*stats.csv +*env/ diff --git a/evals/benchmark/stresscli/README.md b/evals/benchmark/stresscli/README.md new file mode 100644 index 00000000..a2f1bfa1 --- /dev/null +++ b/evals/benchmark/stresscli/README.md @@ -0,0 +1,132 @@ +# StressCli + +This project includes benchmark toolset for AI workloads such as OPEA. + +## stresscli.py + +`stresscli.py` is a command line tool for dumping test specs and performing load tests. + +### Prerequirements + +This tool will use `kubectl` to collect Kubernetes cluster information. So you need to make sure `kubectl` is installed and + +This tool uses `locust` by default to do load test. You have to install `locust` to your machine simply by +``` +pip3 install locust +``` +For detail information of `locust`, go to [locust website](https://docs.locust.io/en/stable/installation.html). + +### Installation + +The recommended way to install and run stresscli is in a virtualenv with the latest version of Python 3 (at least Python 3.11). If Python is not installed, you can likely install it using your distribution's +package manager, or see the [Python Download page](https://www.python.org/downloads/). + +```bash +git clone https://github.com/opea-project/GenAIEval.git +# go to project root +cd GenAIEval/evals/benchmark/stresscli +# create virtual env +python3 -m venv stresscli_virtualenv +source stresscli_virtualenv/bin/activate +# install requirements +pip install -r requirements.txt +``` + +### Usage + +``` +$ ./stresscli.py --help +Usage: stresscli.py [OPTIONS] COMMAND [ARGS]... + + StressCLI - A command line tool for stress testing OPEA workloads. + +Options: + --kubeconfig PATH Configuration file to Kubernetes + --help Show this message and exit. + +Commands: + dump Dump the test spec + load-test Do load test + report Print the test report + validate Validate against the test spec +``` +#### Run a test + +**Note: Please edit the [run.yaml](./run.yaml) file or create your profile before run the load test.** + +``` +./stresscli.py load-test --profile run.yaml +``` + +More detail options: +``` +$ ./stresscli.py load-test --help +Usage: stresscli.py load-test [OPTIONS] + + Do load test + +Options: + --profile PATH Path to profile YAML file + --help Show this message and exit. +``` + +#### Generate the test output report + +You can generate the report for test cases by: +``` +$ ./stresscli.py report --folder /home/sdp/test_reports/20240710_004105 --format csv -o data.csv +``` + +More detail options: +``` +$ ./stresscli.py report --help +Usage: stresscli.py report [OPTIONS] + + Print the test report + +Options: + --folder PATH Path to log folder [required] + --format TEXT Output format, plain_text or csv, default is plain_text + --include TEXT Extract output data from output.log, stats.csv, and + testspec.yaml, default is + output.log|stats.csv|testspec.yaml + -o, --output PATH Save output to file + --help Show this message and exit. +``` +#### Dump the configuration + +You can dump the current testing profile by +``` +./stresscli.py dump -o +``` +More detail options: +``` +$ ./stresscli.py dump --help +Usage: stresscli.py dump [OPTIONS] + + Dump the test spec + +Options: + -o, --output PATH YAML file of cluster configuration [required] + --help Show this message and exit. +``` + +#### Validate against the spec + +You can validate if the current K8s and workloads deployment comply with the test spec by: +``` +$ ./stresscli.py validate --file testspec.yaml +``` + +More detail options: +``` +$ ./stresscli.py validate --help +Usage: stresscli.py validate [OPTIONS] + + Validate against the test spec + +Options: + --file PATH Specification YAML file to validate against [required] + --validate_topology Validate topology in workload specification + --help Show this message and exit. +``` diff --git a/evals/benchmark/stresscli/__init__.py b/evals/benchmark/stresscli/__init__.py new file mode 100644 index 00000000..916f3a44 --- /dev/null +++ b/evals/benchmark/stresscli/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/evals/benchmark/stresscli/commands/__init__.py b/evals/benchmark/stresscli/commands/__init__.py new file mode 100644 index 00000000..916f3a44 --- /dev/null +++ b/evals/benchmark/stresscli/commands/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/evals/benchmark/stresscli/commands/dump.py b/evals/benchmark/stresscli/commands/dump.py new file mode 100644 index 00000000..00210e27 --- /dev/null +++ b/evals/benchmark/stresscli/commands/dump.py @@ -0,0 +1,27 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# stresscli/dump.py + +import click + +from .utils import dump_k8s_config + + +@click.command() +# @click.option('--kubeconfig', type=click.Path(), help='Configuration file to Kubernetes') +@click.option( + "-o", + "--output", + type=click.Path(), + default="testspec.yaml", + required=True, + help="YAML file of cluster configuration", +) +@click.pass_context +#'-o', '--output', type=click.Path(), default='output.yaml', help='YAML file of cluster configuration' +def dump(ctx, output): + """Dump the test spec.""" + kubeconfig = ctx.parent.params["kubeconfig"] + namespace = ctx.parent.params["namespace"] + dump_k8s_config(kubeconfig, output, namespace=namespace) diff --git a/evals/benchmark/stresscli/commands/load_test.py b/evals/benchmark/stresscli/commands/load_test.py new file mode 100644 index 00000000..2915c7bd --- /dev/null +++ b/evals/benchmark/stresscli/commands/load_test.py @@ -0,0 +1,238 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# stresscli/load_test.py + +import os +import subprocess +from datetime import datetime + +import click +import yaml + +from .metrics import MetricsCollector +from .metrics_util import export_metric +from .report import export_testdata +from .utils import dump_k8s_config, generate_lua_script, generate_random_suffix + +# Define default values +locust_defaults = { + "locustfile": "locustfile.py", + "host": "http://127.0.0.1:8888", + "stop-timeout": 120, + "run-time": "48h", + "processes": 2, + "bench-target": "chatqnafixed", + "users": 10, + "max-request": 100, + "namespace": "default", +} + + +@click.command() +# @click.option('--dataset', type=click.Path(), help='Dataset path') +# @click.option('--endpoint', type=click.Path(), help='Endpoint of the test target service, like "http://192.168.0.12:8888/chatqna"') +@click.option("--profile", type=click.Path(), help="Path to profile YAML file") +@click.pass_context +# @click.option('--kubeconfig', type=click.Path(), help='Configuration file to Kubernetes') +def load_test(ctx, profile): + """Do load test.""" + kubeconfig = ctx.parent.params["kubeconfig"] + locust_runtests(kubeconfig, profile) + + +def locust_runtests(kubeconfig, profile): + if profile: + with open(profile, "r") as file: + profile_data = yaml.safe_load(file) + + # create test log folder + hostpath = profile_data["profile"]["storage"]["hostpath"] + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + base_folder = os.path.join(hostpath, f"{timestamp}") + os.makedirs(base_folder, exist_ok=True) + + # Extract storage path and run details from profile + global_settings = profile_data["profile"]["global-settings"] + runs = profile_data["profile"]["runs"] + + index = 1 + for run in runs: + print(f"===Starting test: {run['name']}") + run_locust_test(kubeconfig, global_settings, run, base_folder, index) + print(f"===Completed test: {run['name']}") + index = index + 1 + + click.echo(f"Load test results saved to {base_folder}") + + +def collect_metrics(collector, namespace, services, output_dir): + collector.start_collecting_data( + namespace=namespace, + services=services, + output_dir=output_dir, + restart_pods_flag=False, + ) + + +def run_locust_test(kubeconfig, global_settings, run_settings, output_folder, index): + # Merge settings with priority: run_settings > global_settings > defaults + runspec = {} + runspec["tool"] = "locust" + runspec["locustfile"] = run_settings.get( + "locustfile", global_settings.get("locustfile", locust_defaults["locustfile"]) + ) + runspec["host"] = run_settings.get("host", global_settings.get("host", locust_defaults["host"])) + runspec["runtime"] = run_settings.get("run-time", global_settings.get("run-time", locust_defaults["run-time"])) + runspec["users"] = run_settings.get("users", global_settings.get("users", locust_defaults["users"])) + runspec["max_requests"] = run_settings.get( + "max-request", global_settings.get("max-request", locust_defaults["max-request"]) + ) + runspec["stop_timeout"] = run_settings.get( + "stop-timeout", global_settings.get("stop-timeout", locust_defaults["stop-timeout"]) + ) + runspec["processes"] = run_settings.get("processes", global_settings.get("processes", locust_defaults["processes"])) + runspec["bench-target"] = run_settings.get( + "bench-target", global_settings.get("bench-target", locust_defaults["bench-target"]) + ) + runspec["namespace"] = run_settings.get("namespace", global_settings.get("namespace", locust_defaults["namespace"])) + + runspec["run_name"] = run_settings["name"] + # csv_output = os.path.join(output_folder, runspec['run_name']) + # json_output = os.path.join(output_folder, f"{runspec['run_name']}_output.log") + csv_output = os.path.join(output_folder, f"{index}") + json_output = os.path.join(output_folder, f"{index}_output.log") + + service_metric = global_settings.get("service-metric-collect", False) + if service_metric: + metrics_output_folder = os.path.join(output_folder, f"{index}_metrics") + os.makedirs(metrics_output_folder, exist_ok=True) + start_output_folder = os.path.join(metrics_output_folder, "start") + os.makedirs(start_output_folder, exist_ok=True) + end_output_folder = os.path.join(metrics_output_folder, "end") + os.makedirs(end_output_folder, exist_ok=True) + metrics_output = os.path.join(output_folder, f"{index}_metrics.json") + + cmd = [ + "locust", + "--locustfile", + runspec["locustfile"], + "--host", + runspec["host"], + "--run-time", + runspec["runtime"], + "--users", + str(runspec["users"]), + "--spawn-rate", + str(runspec["users"]), + "--max-request", + str(runspec["max_requests"]), + "--processes", + str(runspec["processes"]), + "--bench-target", + str(runspec["bench-target"]), + "--stop-timeout", + str(runspec["stop_timeout"]), + "--csv", + csv_output, + "--headless", + "--only-summary", + "--loglevel", + "WARNING", + "--json", + ] + + print(f"Running test: {' '.join(cmd)}") + namespace = runspec["namespace"] + + if service_metric: + collector = MetricsCollector() + services = global_settings.get("service-list") or [] + collect_metrics(collector, namespace, services, start_output_folder) + + runspec["starttest_time"] = datetime.now().isoformat() + result = subprocess.run(cmd, capture_output=True, text=True) + runspec["endtest_time"] = datetime.now().isoformat() + + if service_metric: + collect_metrics(collector, namespace, services, end_output_folder) + export_metric(start_output_folder, end_output_folder, metrics_output_folder, metrics_output, services) + + with open(json_output, "w") as json_file: + json_file.write(result.stderr) + json_file.write(result.stdout) + print(result.stderr) + print(result.stdout) + dump_test_spec(kubeconfig, runspec, runspec["namespace"], output_folder, index) + + +def dump_test_spec(kubeconfig, run, namespace, output_folder, index): + # Dump the k8s spec + k8s_spec = dump_k8s_config(kubeconfig, return_as_dict=True, namespace=namespace) + # Check if k8s_spec contains the expected keys + hardware_spec = k8s_spec.get("hardwarespec", {}) + workload_spec = k8s_spec.get("workloadspec", {}) + + data_output = export_testdata(f"{index}", output_folder, "output.log|stats.csv") + # Combine both specs into a single YAML file + combined_spec = { + "benchmarkspec": run, + "benchmarkresult": data_output, + "hardwarespec": hardware_spec, + "workloadspec": workload_spec, + } + combined_spec_path = os.path.join(output_folder, f"{index}_testspec.yaml") + with open(combined_spec_path, "w") as combined_file: + yaml.dump(combined_spec, combined_file) + + +def wrk_runtests(kubeconfig, dataset, endpoint, profile): + if dataset: + click.echo(f"Using dataset: {dataset}") + + # Here you would add the logic to perform the load test + if profile: + with open(profile, "r") as file: + profile_data = yaml.safe_load(file) + + # create test log folder + hostpath = profile_data["profile"]["storage"]["hostpath"] + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + base_folder = os.path.join(hostpath, f"{timestamp}") + os.makedirs(base_folder, exist_ok=True) + + # Extract storage path and run details from profile + runs = profile_data["profile"]["runs"] + + for run in runs: + run_name = run["name"] + namespace = run.get("namespace", "default") + loop = run.get("loop", 1) + tool = run.get("tool", "wrk") + options = run["options"] + endpoint = run["endpoint"] + dataset = run.get("dataset", dataset) + lua_template_path = run.get( + "luatemplate", "/home/sdp/workspace/OPEAStress/stresscli/lua_template/chatqna.template" + ) + + # Create the folder for the run + run_folder = os.path.join(base_folder, f"{run_name}") + os.makedirs(run_folder, exist_ok=True) + + # Generate Lua script + lua_script_path = os.path.join(run_folder, "wrk_script.lua") + generate_lua_script(lua_template_path, lua_script_path, dataset) + + for i in range(loop): + # Generate wrk command and execute it + wrk_command = f"{tool} {options} --script {lua_script_path} {endpoint}" + print(wrk_command) + wrk_log_path = os.path.join(run_folder, f"wrk_{i + 1}.log") + with open(wrk_log_path, "w") as wrk_log: + subprocess.run(wrk_command.split(), stdout=wrk_log, stderr=subprocess.STDOUT) + + dump_test_spec(kubeconfig, run, namespace, run_folder) + click.echo(f"Load test results saved to {base_folder}") + else: + click.echo("Profile is required to run the test.") diff --git a/evals/benchmark/stresscli/commands/main.py b/evals/benchmark/stresscli/commands/main.py new file mode 100644 index 00000000..56fd3a9b --- /dev/null +++ b/evals/benchmark/stresscli/commands/main.py @@ -0,0 +1,27 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# stresscli/main.py + +import click +from commands.dump import dump +from commands.load_test import load_test +from commands.report import report +from commands.validate import validate + + +@click.group() +@click.option("--kubeconfig", type=click.Path(), help="Configuration file to Kubernetes") +@click.option("--namespace", default="default", help="Namespace to dump Kubernetes config from") +def cli(kubeconfig, namespace): + """StressCLI - A command line tool for stress testing OPEA workloads.""" + pass + + +cli.add_command(dump) +cli.add_command(load_test) +cli.add_command(validate) +cli.add_command(report) + +if __name__ == "__main__": + cli() diff --git a/evals/benchmark/stresscli/commands/metrics.py b/evals/benchmark/stresscli/commands/metrics.py new file mode 100644 index 00000000..cfe1948f --- /dev/null +++ b/evals/benchmark/stresscli/commands/metrics.py @@ -0,0 +1,122 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging +import os +import time + +import requests +from kubernetes import client, config + +# Setup logs +log_level = os.getenv("LOG_LEVEL", "ERROR").upper() +logging.basicConfig(level=getattr(logging, log_level)) + +# Load Kubernetes config +config.load_kube_config() + +# Create Kubernetes Client +v1 = client.CoreV1Api() + + +class MetricsCollector: + def __init__(self): + self.v1 = client.CoreV1Api() + + def get_service_selector(self, namespace, service_name): + service = self.v1.read_namespaced_service(service_name, namespace) + return service.spec.selector + + def get_pod_names_and_ips(self, namespace, labels): + label_selector = ",".join([f"{key}={value}" for key, value in labels.items()]) + logging.debug(f"Listing pods in namespace: {namespace} with label: {label_selector}") + pods = self.v1.list_namespaced_pod(namespace, label_selector=label_selector) + pod_info = [(pod.metadata.name, pod.status.pod_ip) for pod in pods.items] + logging.debug(f"Found pods: {pod_info}") + return pod_info + + def restart_pods(self, namespace, labels): + pod_names = [name for name, _ in self.get_pod_names_and_ips(namespace, labels)] + for pod_name in pod_names: + logging.info(f"Restarting pod {pod_name} in namespace {namespace}") + self.v1.delete_namespaced_pod(pod_name, namespace) + logging.info(f"Pod {pod_name} has been restarted.") + + def wait_for_pods(self, namespace, labels, timeout=300): + label_selector = ",".join([f"{key}={value}" for key, value in labels.items()]) + end_time = time.time() + timeout + while time.time() < end_time: + pods = self.v1.list_namespaced_pod(namespace, label_selector=label_selector) + if all(pod.status.phase == "Running" for pod in pods.items): + logging.info("All pods are running.") + return True + time.sleep(5) + logging.error("Timeout waiting for pods to be running.") + return False + + def get_pod_port(self, pod): + port = pod.spec.containers[0].ports[0].container_port + logging.debug(f"Using port {port} for pod {pod.metadata.name}") + return port + + def collect_metrics(self, pod_ip, pod_port, metrics_path): + service_url = f"http://{pod_ip}:{pod_port}{metrics_path}" + logging.debug(f"Collecting metrics from {service_url}") + try: + response = requests.get(service_url) + response.raise_for_status() + logging.debug(f"Metrics collected from {service_url}") + return response.text + except requests.RequestException as e: + logging.error(f"Error collecting metrics from {service_url}: {e}") + return None + + def start_collecting_data(self, namespace, services, output_dir="/data", restart_pods_flag=True): + timestamp = int(time.time()) + metrics_path = "/metrics" + + for service_name in services: + service_labels = self.get_service_selector(namespace, service_name) + + pod_infos = self.get_pod_names_and_ips(namespace, service_labels) + + for pod_name, pod_ip in pod_infos: + pod_info = self.v1.read_namespaced_pod(name=pod_name, namespace=namespace) + pod_port = self.get_pod_port(pod_info) + metrics = self.collect_metrics(pod_ip, pod_port, metrics_path) + if metrics: + pod_output_path = os.path.join(output_dir, f"{service_name}_{pod_name}_{timestamp}.txt") + logging.debug(f"Writing metrics to {pod_output_path}") + with open(pod_output_path, "w") as f: + f.write(metrics) + else: + logging.error(f"No metrics collected for pod {pod_name}") + + # Restart pods if the flag is set + if restart_pods_flag: + self.restart_pods(namespace, service_labels) + logging.info(f"Pods for service {service_name} have been restarted.") + + # Wait for Service Pods Running after restart + if restart_pods_flag: + for service_name in services: + service_labels = self.get_service_selector(namespace, service_name) + if not self.wait_for_pods(namespace, service_labels): + logging.error(f"Pods for service {service_name} did not become ready in time.") + return { + "status": "error", + "message": f"Pods for service {service_name} did not become ready in time.", + } + + return {"status": "success"} + + +if __name__ == "__main__": + collector = MetricsCollector() + result = collector.start_collecting_data( + namespace="chatqna", + services=["chatqna-tgi", "chatqna-tei", "chatqna-teirerank"], + output_dir="/path/to/data", + restart_pods_flag=False, + ) + print(result) diff --git a/evals/benchmark/stresscli/commands/metrics_util.py b/evals/benchmark/stresscli/commands/metrics_util.py new file mode 100644 index 00000000..1763eea5 --- /dev/null +++ b/evals/benchmark/stresscli/commands/metrics_util.py @@ -0,0 +1,204 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import re +from collections import defaultdict + +# Setup logs +log_level = os.getenv("LOG_LEVEL", "ERROR").upper() +logging.basicConfig(level=getattr(logging, log_level)) + + +def parse_metrics(file_path): + logging.debug(f"Parsing metrics from {file_path}") + metrics = {} + with open(file_path, "r") as f: + for line in f: + # Ignore comments and empty lines + if line.startswith("#") or not line.strip(): + continue + # Split metric name and value + try: + metric, value = line.rsplit(" ", 1) + value = value.strip() + if "." in value: + metrics[metric.strip()] = float(value) + else: + metrics[metric.strip()] = int(value) + except ValueError: + logging.warning(f"Skipping invalid line: {line.strip()}") + logging.debug(f"Parsed metrics: {metrics}") + return metrics + + +def write_metrics(file_path, metrics, service_name): + logging.debug(f"Writing metrics to {file_path}") + new_metrics = {} + with open(file_path, "w") as f: + for metric, value in metrics.items(): + f.write(f"{metric} {value}\n") + if "count" in metric and "request_duration_count" not in metric: + f.write("\n") + # Handle TGI/TEI + if "request_duration_sum" in metric: + count_metric = metric.replace("request_duration_sum", "request_duration_count") + logging.info(f"match {metric} and {count_metric}") + if count_metric in metrics and metrics[count_metric] != 0: + average_latency = metrics[metric] / metrics[count_metric] + new_metric = f"{service_name}_average_latency_per_request" + new_metrics[new_metric] = average_latency + f.write(f"{new_metric} {average_latency}\n") + f.write("\n") + # Handle OPEA microservices + if "http_request_duration_seconds_sum" in metric: + # Extract the part after '/v1/' and before comma as handler part + match = re.search(r'\{handler="/v1/([^"]*?)"', metric) + if match: + handler_part = match.group(1) # Extract the part after '/v1/' and before comma + + # Construct prefix + sum_metric_prefix = f'http_request_duration_seconds_sum{{handler="/v1/{handler_part}"' + count_metric_prefix = f'http_request_duration_seconds_count{{handler="/v1/{handler_part}"' + + # Find the key begin with prefix and end with '}' + sum_metric_key = next( + (key for key in metrics if key.startswith(sum_metric_prefix) and key.endswith("}")), None + ) + count_metric_key = next( + (key for key in metrics if key.startswith(count_metric_prefix) and key.endswith("}")), None + ) + + if sum_metric_key and count_metric_key: + metric_value = metrics.get(sum_metric_key, "Not Found") + count_metric_value = metrics.get(count_metric_key, "Not Found") + logging.info( + f"Match {sum_metric_key}: {metric_value} and {count_metric_key}: {count_metric_value}" + ) + + if count_metric_value != "Not Found" and count_metric_value != 0: + average_latency = metric_value / count_metric_value + new_metric = f"{service_name}_average_latency_per_request" + new_metrics[new_metric] = average_latency + f.write(f"{new_metric} {average_latency}\n") + f.write("\n") + logging.info(f"{new_metric} {average_latency}") + else: + logging.info(f"Count metric {count_metric_key} is either not found or zero.") + else: + logging.info("Could not find matching metrics for sum or count.") + else: + logging.info("Handler part not found in metric.") + else: + logging.info("Metric does not contain 'http_request_duration_seconds_sum'.") + + return new_metrics + + +def calculate_diff(start_dir, end_dir, output_dir, services=None): + logging.debug(f"Calculating diff between {start_dir} and {end_dir}") + + if services is None: + logging.info("services is None, exiting.") + return [] + + start_files = os.listdir(start_dir) + end_files = os.listdir(end_dir) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + average_latency_metrics = [] + + if isinstance(services, str): + services = [services] + + for service_name in services: + # Create a regex pattern to match files starting with the service_name followed by a non-alphanumeric character + pattern = rf"^{re.escape(service_name)}[^a-zA-Z].*\.txt$" + + start_service_files = [f for f in start_files if re.match(pattern, f)] + end_service_files = [f for f in end_files if re.match(pattern, f)] + + start_metrics = defaultdict(int) + end_metrics = defaultdict(int) + + for file_name in start_service_files: + file_path = os.path.join(start_dir, file_name) + metrics = parse_metrics(file_path) + for metric, value in metrics.items(): + start_metrics[metric] += value + + for file_name in end_service_files: + file_path = os.path.join(end_dir, file_name) + metrics = parse_metrics(file_path) + for metric, value in metrics.items(): + end_metrics[metric] += value + + # Calculate the difference + diff_metrics = {} + for metric in end_metrics: + if metric in start_metrics: + if isinstance(end_metrics[metric], int) and isinstance(start_metrics[metric], int): + diff_value = end_metrics[metric] - start_metrics[metric] + elif isinstance(end_metrics[metric], float) and isinstance(start_metrics[metric], float): + diff_value = end_metrics[metric] - start_metrics[metric] + else: + logging.warning(f"Type mismatch for metric {metric}, skipping") + continue + diff_metrics[metric] = diff_value + + logging.debug(f"Difference metrics for service {service_name}: {diff_metrics}") + + # Write the diff metrics to the output file + output_path = os.path.join(output_dir, f"{service_name}.diff") + new_metrics = write_metrics(output_path, diff_metrics, service_name) + + # Collect average latency metrics + for metric, value in new_metrics.items(): + if "average_latency" in metric: + average_latency_metrics.append((metric, value)) + + logging.info(f"Diff calculated and saved to {output_path}") + + return average_latency_metrics + + +def export_metric(start_dir, end_dir, output_dir, json_output, services): + average_latency_metrics = calculate_diff(start_dir, end_dir, output_dir, services) + + if not average_latency_metrics: + logging.info("No average latency metrics to export, exiting.") + return + + if os.path.exists(json_output): + with open(json_output, "r") as json_file: + try: + existing_data = json.load(json_file) + except json.JSONDecodeError: + logging.warning(f"Existing JSON file {json_output} is empty or invalid, starting fresh") + existing_data = {} + else: + existing_data = {} + + for metric, value in average_latency_metrics: + existing_data[metric] = value + print(f"{metric}: {value}") + + with open(json_output, "w") as json_file: + json.dump(existing_data, json_file, indent=4) + json_file.write("\n") + + logging.info(f"Average latency metrics written to {json_output}") + + +if __name__ == "__main__": + start_dir = "/path/to/data/start" + end_dir = "/path/to/data/end" + output_dir = "/path/to/data" + services = ["chatqna-tgi", "chatqna-tei", "chatqna-teirerank", "chatqna-xeon-backend"] + json_output = "/path/to/data/metrics.log" + + export_metric(start_dir, end_dir, output_dir, json_output, services) diff --git a/evals/benchmark/stresscli/commands/report.py b/evals/benchmark/stresscli/commands/report.py new file mode 100644 index 00000000..600bb502 --- /dev/null +++ b/evals/benchmark/stresscli/commands/report.py @@ -0,0 +1,221 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# stresscli/dump.py + +import configparser +import csv +import glob +import os +import re +import sys + +import click +import yaml + +KEYWORDS_SECTION_NAME = "output.log" +CSV_SECTION_NAME = "stats.csv" +TESTSPEC_SECTION_NAME = "testspec.yaml" +METRICS_SECTION_NAME = "metrics.json" + + +@click.command() +@click.option("--folder", type=click.Path(), required=True, help="Path to log folder") +@click.option("--format", default="plain_text", help="Output format, plain_text or csv, default is plain_text") +# @click.option('--include', default='testspec.yaml', help='Extract output data from output.log, stats.csv, and testspec.yaml, default is testspec.yaml') +@click.option("-o", "--output", type=click.Path(), help="Save output to file") +@click.pass_context +def report(ctx, folder, format, output): + """Print the test report.""" + output_data = {} + testcases = get_testcases(folder) + for testcase in testcases: + include = "|".join([TESTSPEC_SECTION_NAME, CSV_SECTION_NAME, METRICS_SECTION_NAME]) + extracted_data = export_testdata(testcase, folder, include) + if extracted_data: + output_data[testcase] = extracted_data + + if format == "plain_text": + if output: + with open(output, "w") as f: + for testcase, data in output_data.items(): + f.write(f"Testcase: {testcase} {data['run_name']}\n") + for key, value in data.items(): + f.write(f" {key}: {value}\n") + f.write("\n") + else: + for testcase, data in output_data.items(): + print(f"Testcase: {testcase} {data['run_name']}") + for key, value in data.items(): + print(f" {key}: {value}") + print() + + elif format == "csv": + headers = ["No"] + for data in output_data.values(): + for key in data.keys(): + if key not in headers: + headers.append(key) + + if output: + with open(output, "w", newline="") as csvfile: + csvwriter = csv.DictWriter(csvfile, fieldnames=headers) + csvwriter.writeheader() + for testcase, data in output_data.items(): + row = {"No": testcase} + row.update(data) + csvwriter.writerow(row) + else: + csvwriter = csv.DictWriter(sys.stdout, fieldnames=headers) + csvwriter.writeheader() + for testcase, data in output_data.items(): + row = {"No": testcase} + row.update(data) + csvwriter.writerow(row) + + +def export_testspec(testcase, folder): + testspec_path = os.path.join(folder, f"{testcase}_testspec.yaml") + extracted_data = {} + if os.path.exists(testspec_path): + extract_yaml(extracted_data, testspec_path) + + +def export_testdata(testcase, folder, include="output.log|stats.csv|testspec.yaml|metrics.json"): + output_log = os.path.join(folder, f"{testcase}_output.log") + csv_path = os.path.join(folder, f"{testcase}_stats.csv") + testspec_path = os.path.join(folder, f"{testcase}_testspec.yaml") + metrics_path = os.path.join(folder, f"{testcase}_metrics.json") + extracted_data = {} + if os.path.exists(csv_path): + if TESTSPEC_SECTION_NAME in include: + extract_yaml(extracted_data, testspec_path) + if CSV_SECTION_NAME in include: + extract_csv(extracted_data, csv_path) + if KEYWORDS_SECTION_NAME in include: + with open(output_log, "r") as file: + log_data = file.read() + extract_stdout(extracted_data, log_data) + if METRICS_SECTION_NAME in include and os.path.exists(metrics_path): + extract_json(extracted_data, metrics_path) + else: + print("Test failure, no data") + return extracted_data + + +# Function to read keywords and patterns from config file +def read_log_keywords(filename="config.ini"): + config = configparser.ConfigParser() + config.optionxform = str + config.read(filename) + keywords = {} + if KEYWORDS_SECTION_NAME in config: + for key, pattern in config[KEYWORDS_SECTION_NAME].items(): + keywords[key] = pattern + return keywords + + +def read_csv_keywords(filename="config.ini"): + config = configparser.ConfigParser(interpolation=None) + config.read(filename) + columns_to_extract = [col.strip() for col in config.get(CSV_SECTION_NAME, "columns_name").split(",")] + row_name = config.get(CSV_SECTION_NAME, "row_name").strip() + return columns_to_extract, row_name + + +def read_yaml_keywords(filename="config.ini"): + config = configparser.ConfigParser() + config.optionxform = str + config.read(filename) + keywords = {} + if TESTSPEC_SECTION_NAME in config: + for key, pattern in config[TESTSPEC_SECTION_NAME].items(): + keywords[key] = pattern + return keywords + + +import json + + +def extract_json(extracted_data, json_file): + try: + with open(json_file, "r") as file: + data = json.load(file) + + for key, value in data.items(): + print(f"Key: {key}, Value: {value}") + extracted_data[key] = value + + except json.JSONDecodeError as e: + print(f"Error decoding JSON: {e}") + except FileNotFoundError: + print("The specified file was not found.") + except Exception as e: + print(f"An error occurred: {e}") + + +# Function to extract information based on keywords and patterns +def extract_stdout(extracted_data, log): + keywords = read_log_keywords("config.ini") + for key, pattern in keywords.items(): + match = re.search(pattern, log) + if match: + extracted_data[key] = match.group(1) + # for key, value in extracted_data.items(): + # print(f"{key}: {value}") + return extracted_data + + +def extract_csv(extracted_data, csv_file): + # Read configuration values + columns_to_extract, row_name = read_csv_keywords() + with open(csv_file, newline="") as file: + reader = csv.DictReader(file) + + # Iterate through each row in the CSV file + for row in reader: + # Check if the row corresponds to the second row ("Aggregated" row) + if row["Name"] == row_name: + # Extract the values for the specified columns + for column in columns_to_extract: + # extracted_data[f'locust_P{column[:-1]}'] = float(row[column]) + extracted_data[f"locust_P{column[:-1]}"] = row[column] + + # Once values are extracted, break out of the loop + break # Print the extracted values + + +def extract_yaml(extracted_data, yaml_file): + # Read configuration values + keywords = read_yaml_keywords("config.ini") + # Load YAML file + with open(yaml_file, "r") as f: + yaml_data = yaml.safe_load(f) + extracted_data["run_name"] = yaml_data["benchmarkspec"]["run_name"] + for section, pattern in keywords.items(): + if section in yaml_data: + if pattern == "*": + extracted_data.update(yaml_data[section]) + else: + attributes = pattern.split() + for attr in attributes: + if attr in yaml_data[section]: + extracted_data[f"{attr}"] = yaml_data[section][attr] + else: + for key, value in yaml_data[section].items(): + if isinstance(value, dict) and attr in value: + extracted_data[f"{attr}"] = value[attr] + break + + +def get_testcases(directory_path): + yaml_files = glob.glob(os.path.join(directory_path, "*.yaml")) + first_parts = [] + + for yaml_file in yaml_files: + file_name = os.path.basename(yaml_file) + file_name_parts = file_name.split("_") + first_part = file_name_parts[0] + first_parts.append(first_part) + + return sorted(first_parts) diff --git a/evals/benchmark/stresscli/commands/utils.py b/evals/benchmark/stresscli/commands/utils.py new file mode 100644 index 00000000..19fd86b1 --- /dev/null +++ b/evals/benchmark/stresscli/commands/utils.py @@ -0,0 +1,198 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# stresscli/utils.py + +import json +import random +import string +import subprocess + +import yaml +from kubernetes import client +from kubernetes import config as k8s_config + + +def dump_k8s_config(kubeconfig, output=None, return_as_dict=False, namespace="default"): + """Dump the Kubernetes cluster configuration to a YAML file.""" + # Load Kubernetes configuration + if kubeconfig: + k8s_config.load_kube_config(config_file=kubeconfig) + else: + k8s_config.load_kube_config() + + # Initialize the Kubernetes API client + v1 = client.CoreV1Api() + apps_v1 = client.AppsV1Api() + + # Get all nodes in the cluster + nodes = v1.list_node() + node_deployments = {} + node_hardware_spec = {} + + # excluded_namespaces = {"kube-system", "kube-public", "kube-node-lease", "local-path-storage"} + for node in nodes.items: + node_name = node.metadata.name + # Generate hardwarespec + + node_info = node.status.node_info + node_hardware_spec[node_name] = { + "architecture": node_info.architecture, + "containerRuntimeVersion": node_info.container_runtime_version, + "kernelVersion": node_info.kernel_version, + "kubeProxyVersion": node_info.kube_proxy_version, + "kubeletVersion": node_info.kubelet_version, + "operatingSystem": node_info.operating_system, + "osImage": node_info.os_image, + } + if "calico" in get_k8s_cni(v1): + node_hardware_spec[node_name]["canico"] = get_calico_version() + if get_cpu_manager_policy(v1) != "none": + node_hardware_spec["cpuManagerPolicy"] = "static" + node_capacity = {k: str(v) for k, v in node.status.capacity.items()} + if "cpu" in node_capacity: + node_hardware_spec[node_name]["cpu"] = node_capacity["cpu"] + if "memory" in node_capacity: + node_hardware_spec[node_name]["memory"] = node_capacity["memory"] + if "habana.ai/gaudi" in node_capacity: + node_hardware_spec[node_name]["habana.ai/gaudi"] = node_capacity["habana.ai/gaudi"] + + # Generate workloadspec + node_deployments[node_name] = {} + # Get all pods running on the node + field_selector = f"spec.nodeName={node_name}" + # pods = v1.list_pod_for_all_namespaces(field_selector=field_selector) + pods = v1.list_namespaced_pod(namespace=namespace, field_selector=field_selector) + + for pod in pods.items: + # if pod.metadata.namespace in excluded_namespaces: + # continue + owner_references = pod.metadata.owner_references + for owner in owner_references: + if owner.kind == "ReplicaSet": + replicaset_name = owner.name + # Get the deployment owning the replicaset + rs = apps_v1.read_namespaced_replica_set(name=replicaset_name, namespace=pod.metadata.namespace) + for owner in rs.metadata.owner_references: + if owner.kind == "Deployment": + deployment_name = owner.name + if deployment_name not in node_deployments[node_name]: + node_deployments[node_name][deployment_name] = {"replica": 0} + node_deployments[node_name][deployment_name]["replica"] += 1 + # Collect resource information + resources = pod.spec.containers[0].resources + resource_details = {} + if resources.requests: + resource_details["requests"] = resources.requests + if resources.limits: + resource_details["limits"] = resources.limits + if resource_details: + node_deployments[node_name][deployment_name]["resources"] = resource_details + + k8s_spec = {"hardwarespec": node_hardware_spec, "workloadspec": node_deployments} + if return_as_dict: + return k8s_spec + else: + # Write the deployment details to the YAML file + with open(output, "w") as yaml_file: + yaml.dump(k8s_spec, yaml_file, default_flow_style=False) + print(f"Cluster configuration written to {output}") + + +def generate_random_suffix(length=6): + """Generate a random string of fixed length.""" + letters = string.ascii_lowercase + string.digits + return "".join(random.choice(letters) for _ in range(length)) + + +def generate_lua_script(template_path, output_path, dataset_path): + """Generate a Lua script from the template with the specified dataset path.""" + with open(template_path, "r") as template_file: + template_content = template_file.read() + + script_content = template_content.replace("${DATASET_PATH}", dataset_path) + + with open(output_path, "w") as output_file: + output_file.write(script_content) + + +def run_command(command): + """Runs a command and returns the output.""" + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, text=True) + if result.returncode != 0: + print(f"Error running command: {result.stderr}") + return None + return result.stdout.strip() + + +# get cni version +def get_calico_version(): + command = [ + "kubectl", + "get", + "pods", + "-n", + "kube-system", + "-l", + "k8s-app=calico-node", + "-o", + "jsonpath='{.items[*].spec.containers[*].image}'", + ] + image_name = run_command(command) + # print(image_name) + parts = image_name.split(":") + if len(parts) > 1: + return parts[-1] + else: + return None + + +def get_k8s_cni(v1): + # List all pods in the kube-system namespace + pods = v1.list_namespaced_pod(namespace="kube-system") + + cni_plugins = { + "calico": "calico-node", + "flannel": "kube-flannel-ds", + "weave": "weave-net", + "cilium": "cilium", + "contiv": "contiv-netplugin", + "canal": "canal", + "romana": "romana-agent", + "kube-router": "kube-router", + "multus": "kube-multus-ds", + } + + detected_plugins = [] + for pod in pods.items: + for cni, identifier in cni_plugins.items(): + if identifier in pod.metadata.name: + detected_plugins.append(cni) + # if detected_plugins: + # print(f"Detected CNI Plugins: {', '.join(detected_plugins)}") + # else: + # print("No known CNI plugin detected.") + return detected_plugins + + +def get_cpu_manager_policy(v1): + # Get the kubelet-config ConfigMap + try: + config_map = v1.read_namespaced_config_map(name="kubelet-config", namespace="kube-system") + except client.exceptions.ApiException as e: + print(f"Exception when retrieving ConfigMap: {e}") + return + + # Parse the ConfigMap data + kubelet_config_data = config_map.data.get("kubelet") + if not kubelet_config_data: + print("kubelet configuration not found in the ConfigMap") + return + + # Load the kubelet configuration YAML + kubelet_config = yaml.safe_load(kubelet_config_data) + + # Get the cpuManagerPolicy setting + cpu_manager_policy = kubelet_config.get("cpuManagerPolicy", "none") + # print(f"CPU Manager Policy: {cpu_manager_policy}") + return cpu_manager_policy diff --git a/evals/benchmark/stresscli/commands/validate.py b/evals/benchmark/stresscli/commands/validate.py new file mode 100644 index 00000000..a8ffb6b9 --- /dev/null +++ b/evals/benchmark/stresscli/commands/validate.py @@ -0,0 +1,164 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# stresscli/validate.py + +import difflib +import json + +import click +import yaml + +from .utils import dump_k8s_config + + +@click.command() +@click.option("--file", type=click.Path(), required=True, help="Specification YAML file to validate against") +@click.option("--validate_topology", is_flag=True, help="Validate topology in workload specification") +@click.pass_context +def validate(ctx, file, validate_topology): + """Validate against the test spec.""" + kubeconfig = ctx.parent.params["kubeconfig"] + namespace = ctx.parent.params["namespace"] + current_state = dump_k8s_config(kubeconfig, return_as_dict=True, namespace=namespace) + validate_spec(kubeconfig, file, current_state, validate_topology) + + +def read_spec(file_path): + try: + with open(file_path, "r") as file: + return yaml.safe_load(file) + except FileNotFoundError: + print(f"File not found: {file_path}") + return None # Or some other default value or action + + +# hardwarespec: +# mycluster-control-plane: +# cpu: '224' +# memory: 263835592Ki +# workloadspec: +# node1: +# chatqna: +# replica: 1 +# chatqna-embedding-usvc: +# replica: 1 +def validate_spec(kubeconfig, spec_file, current_state, validate_topology=False): + spec = read_spec(spec_file) + if spec is None: + return + + errors = [] + # Validate hardware spec and issue warnings + for node_spec in spec.get("hardwarespec", {}).values(): + matched = False + for current_node_spec in current_state["hardwarespec"].values(): + match = True + warnings = [] + for key, value in node_spec.items(): + if key in current_node_spec: + if key == "cpu" or key == "habana.ai/gaudi": + try: + if int(current_node_spec[key]) > int(value): + warnings.append( + f"Warning: Actual {key} ({current_node_spec[key]}) is higher than specified {key} ({value})." + ) + elif int(current_node_spec[key]) < int(value): + errors.append(f"Actual {key} ({current_node_spec[key]}), expected {key} ({value}).") + except ValueError: + pass # Handle non-integer values gracefully + elif key == "memory": + if current_node_spec[key] > value: + warnings.append( + f"Warning: Actual {key} ({current_node_spec[key]}) is higher than specified {key} ({value})." + ) + elif current_node_spec[key] < value: + errors.append(f"Error: Actual {key} ({current_node_spec[key]}), expected {key} ({value}).") + else: + if current_node_spec[key] != value: + warnings.append( + f"Warning: Actual {key} ({current_node_spec[key]}), expected {key} ({value})." + ) + else: + match = False + break + if match: + matched = True + for warning in warnings: + print(warning) + break + if not matched: + errors.append(f"Error: No matching node found for hardware spec: {node_spec}") + + # Validate workload spec + unmatched_workloads = {} + for workloads in spec.get("workloadspec", {}).values(): + matched = False + for current_workloads in current_state["workloadspec"].values(): + unmatched = {} + for workload_name, workload_spec in workloads.items(): + if workload_name not in current_workloads: + unmatched[workload_name] = workload_spec + else: + current_workload_spec = current_workloads.get(workload_name, {}) + if current_workload_spec.get("replica") != workload_spec["replica"]: + unmatched[workload_name] = workload_spec + for ws in workload_spec.get("workloadspec", []): + for cs in current_workload_spec.get("workloadspec", []): + if ws["container"] == cs["container"]: + for key in ["limits", "requests"]: + if ws["resources"].get(key) != cs["resources"].get(key): + unmatched[workload_name] = workload_spec + if not unmatched: + matched = True + break + else: + unmatched_workloads.update(unmatched) + if not matched: + errors.append(f"Error: No matching workloads found for spec: {unmatched_workloads}") + + # Validate topology if required + if validate_topology: + validate_topology_siblings(spec.get("workloadspec", {}), current_state.get("workloadspec", {}), errors) + + if errors: + for error in errors: + print(error) + print("Validation failed.") + print("Get more details of the spec differences:") + diff = compare_dicts(spec, current_state) + print(diff) + else: + print("Validation successful.") + + +def validate_topology_siblings(spec_workload, current_workload, errors): + spec_siblings = get_siblings(spec_workload) + current_siblings = get_siblings(current_workload) + missing_siblings = spec_siblings - current_siblings + # extra_siblings = current_siblings - spec_siblings + + if missing_siblings: + if missing_siblings: + for s in missing_siblings: + errors.append(f"Error in topology: Not found {list(s)} in same K8s node") + + +def get_siblings(workload): + sibling_sets = set() + for workloads in workload.values(): + sibling_sets.add(frozenset(workloads.keys())) + return sibling_sets + + +def dict_to_str(dictionary): + return json.dumps(dictionary, indent=2, sort_keys=True) + + +def compare_dicts(dict1, dict2): + str1 = dict_to_str(dict1) + str2 = dict_to_str(dict2) + diff = difflib.unified_diff( + str1.splitlines(), str2.splitlines(), fromfile="expected_spec", tofile="current_spec", lineterm="" + ) + return "\n".join(diff) diff --git a/evals/benchmark/stresscli/config.ini b/evals/benchmark/stresscli/config.ini new file mode 100644 index 00000000..0c01df9a --- /dev/null +++ b/evals/benchmark/stresscli/config.ini @@ -0,0 +1,27 @@ +[output.log] +# DON'T CHANGE +Succeed_Response = Succeed Response:\s+(\d+) +Duration = Duration:\s+([\d.]+)s +Tokens = Tokens:\s+(\d+) +RPS = RPS:\s+([\d.]+) +Tokens_per_Second = Tokens per Second:\s+([\d.]+) +End_to_End_latency_P50 = End to End latency\(ms\),\s+P50:\s+([\d.]+) +End_to_End_latency_P99 = End to End latency\(ms\),\s+P50:[\s\d.,]+P99:\s+([\d.]+) +End_to_End_latency_Avg = End to End latency\(ms\),\s+P50:[\s\d.,]+P99:\s+[\s\d.,]+Avg:\s+([\d.]+) +First_token_latency_P50 = First token latency\(ms\),\s+P50:\s+([\d.]+) +First_token_latency_P99 = First token latency\(ms\),\s+P50:[\s\d.,]+P99:\s+([\d.]+) +First_token_latency_Avg = First token latency\(ms\),\s+P50:[\s\d.,]+P99:\s+[\s\d.,]+Avg:\s+([\d.]+) +Average_Next_token_latency = Average Next token latency\(ms\):\s+([\d.]+) +Average_token_latency = Average token latency\(ms\)\s+:\s+([\d.]+) +locust_num_requests = \"num_requests\":\s+(\d+) +locust_num_failures = \"num_failures\":\s+(\d+) + +[stats.csv] +# DON'T CHANGE +columns_name = 50%, 99% +row_name = Aggregated + +[testspec.yaml] +benchmarkresult = * +benchmarkspec = users max_requests +hardwarespec = containerRuntimeVersion cpu \ No newline at end of file diff --git a/evals/benchmark/stresscli/dataset/chatqna.json b/evals/benchmark/stresscli/dataset/chatqna.json new file mode 100644 index 00000000..b5728b0d --- /dev/null +++ b/evals/benchmark/stresscli/dataset/chatqna.json @@ -0,0 +1,191 @@ +[ +{"qId": "wqr000000", "qText": "what is the name of justin bieber brother?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "10131898", "text": "monk", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9836176", "text": "associate", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10256643", "text": "kinsman", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9651570", "text": "religious person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10538195", "text": "religious", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10255246", "text": "relative", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9649426", "text": "peer", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10132360", "text": "friend", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10305781", "text": "male sibling", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10326901", "text": "member", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "brother", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Justin Bieber", "cookedLabel": "Justin Bieber", "pageID": "23680998", "editDist": 0.0, "labelProbability": 0.995669, "logPopularity": 5.991464547107982, "score": 0.925242503944315, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the name of justin bieber brother", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "name of justin bieber brother", "type": "CluePhrase", "weight": 0.99}, {"label": "name", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "justin bieber brother", "type": "CluePhrase", "weight": 0.99}] }, +{"qId": "wqr000020", "qText": "where to fly into bali?", "SV": ["fly"], "lemmaSV": ["fly"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "To Fly!", "cookedLabel": "To Fly!", "pageID": "76390", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.174387269895637, "score": 0.8093029754987044, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Bali", "cookedLabel": "Bali", "pageID": "4147", "editDist": 0.0, "labelProbability": 0.882369, "logPopularity": 5.8664680569332965, "score": 0.8721413788784711, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "to fly", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000040", "qText": "what is cher's son's name?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9937706", "text": "child", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9559169", "text": "God", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5950141", "text": "belief", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "10393697", "text": "offspring", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9561132", "text": "hypostasis", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9647338", "text": "male", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10305635", "text": "male offspring", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "9527267", "text": "spiritual being", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "9559474", "text": "Godhead", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10255246", "text": "relative", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "son", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Humphrey de Cherlton", "cookedLabel": "Humphrey de Cherlton", "pageID": "36553055", "editDist": 2.3, "labelProbability": 0.0, "logPopularity": 3.295836866004329, "score": 0.011909595991362784, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Cher", "cookedLabel": "Cher", "pageID": "80696", "editDist": 0.0, "labelProbability": 0.74216, "logPopularity": 6.326149473155099, "score": 0.8251100043767303, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Sonny & Cher", "cookedLabel": "Sonny & Cher", "pageID": "113446", "editDist": 0.0, "labelProbability": 0.0646615, "logPopularity": 4.955827057601261, "score": 0.0332190417502861, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Gypsys, Tramps & Thieves (album)", "cookedLabel": "Gypsys, Tramps & Thieves", "pageID": "4437004", "editDist": 0.0, "labelProbability": 0.0646615, "logPopularity": 4.709530201312334, "score": 0.028786938773518025, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Cher Lloyd", "cookedLabel": "Cher Lloyd", "pageID": "29044071", "editDist": 0.0, "labelProbability": 0.0646615, "logPopularity": 4.7535901911063645, "score": 0.029535308046203673, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Cher (department)", "cookedLabel": "Cher", "pageID": "80697", "editDist": 0.0, "labelProbability": 0.141742, "logPopularity": 5.976350909297934, "score": 0.08284955932906549, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "cher's son's name", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "cher", "type": "ClueNE", "weight": 1.09}, {"label": "name", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr000060", "qText": "what countries do people speak portuguese?", "SV": ["speak"], "lemmaSV": ["speak"], "LAT": [{"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7958392", "text": "people", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "countries", "specificity": "0.0", "type": "LAT"}, {"text": "country", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "People (magazine)", "cookedLabel": "People", "pageID": "507970", "editDist": 0.0, "labelProbability": 0.174827, "logPopularity": 4.584967478670572, "score": 0.10998291320708259, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "People", "cookedLabel": "People", "pageID": "3488351", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.0794415416798357, "score": 0.031159073502995693, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Portuguese language", "cookedLabel": "Portuguese language", "pageID": "23915", "editDist": 0.0, "labelProbability": 0.441879, "logPopularity": 7.486613313139955, "score": 0.7042009586468194, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Sonnets from the Portuguese", "cookedLabel": "Sonnets from the Portuguese", "pageID": "1102758", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.044522437723423, "score": 0.02075436300687559, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Portuguese man o' war", "cookedLabel": "Portuguese man o' war", "pageID": "152952", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.8501476017100584, "score": 0.03322510904474677, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Portuguese people", "cookedLabel": "Portuguese people", "pageID": "970642", "editDist": 0.0, "labelProbability": 0.102717, "logPopularity": 6.543911845564792, "score": 0.22145882746554235, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Portugal", "cookedLabel": "Portugal", "pageID": "23033", "editDist": 0.0, "labelProbability": 0.256593, "logPopularity": 9.178746500385005, "score": 0.5112655077499029, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "portuguese", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr000080", "qText": "who was vincent van gogh inspired by?", "SV": ["inspired"], "lemmaSV": ["inspire"], "LAT": [], "Concept": [{"fullLabel": "Vincent van Gogh", "cookedLabel": "Vincent van Gogh", "pageID": "32603", "editDist": 0.0, "labelProbability": 0.993177, "logPopularity": 6.154858094016418, "score": 0.9807994905546668, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000100", "qText": "when will oscar pistorius compete?", "SV": ["compete"], "lemmaSV": ["compete"], "LAT": [{"synset": "15147173", "text": "time", "specificity": "0.0", "type": "QuestionWordLAT"}, {"synset": "15184543", "text": "date", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Oscar Pistorius", "cookedLabel": "Oscar Pistorius", "pageID": "5729054", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.653960350157523, "score": 0.955394663736244, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000120", "qText": "who plays meg in family guy?", "SV": ["plays"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Meg Griffin", "cookedLabel": "Meg Griffin", "pageID": "723502", "editDist": 0.0, "labelProbability": 0.458351, "logPopularity": 4.127134385045092, "score": 0.2549178022150101, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Meg Ryan", "cookedLabel": "Meg Ryan", "pageID": "51799", "editDist": 0.0, "labelProbability": 0.12473, "logPopularity": 5.017279836814924, "score": 0.11185170037908035, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Meg (singer)", "cookedLabel": "Meg", "pageID": "22474590", "editDist": 0.0, "labelProbability": 0.12473, "logPopularity": 4.77912349311153, "score": 0.09842432271625319, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Meg Tilly", "cookedLabel": "Meg Tilly", "pageID": "667049", "editDist": 0.0, "labelProbability": 0.12473, "logPopularity": 4.74493212836325, "score": 0.09661886993478508, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Meg Whitman", "cookedLabel": "Meg Whitman", "pageID": "741886", "editDist": 0.0, "labelProbability": 0.12473, "logPopularity": 4.836281906951478, "score": 0.10150969453190346, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Family Guy", "cookedLabel": "Family Guy", "pageID": "187586", "editDist": 0.0, "labelProbability": 0.964212, "logPopularity": 6.248042874508429, "score": 0.9458401875550986, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "meg", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr000140", "qText": "what stadium did the chicago cardinals play in?", "SV": ["play"], "lemmaSV": ["play"], "LAT": [{"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4348764", "text": "structure", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "stadium", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Arizona Cardinals", "cookedLabel": "Arizona Cardinals", "pageID": "2102", "editDist": 0.0, "labelProbability": 0.260508, "logPopularity": 7.520776415062797, "score": 0.5985305536890859, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "History of the Arizona Cardinals", "cookedLabel": "History of the Arizona Cardinals", "pageID": "9997766", "editDist": 0.0, "labelProbability": 0.482546, "logPopularity": 3.258096538021482, "score": 0.24275810950836152, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "History of the Chicago Cardinals", "cookedLabel": "History of the Chicago Cardinals", "pageID": "34268853", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 5.583496308781699, "score": 0.2690152293647376, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "chicago cardinals", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr000160", "qText": "who was the apostle paul considered to be?", "SV": ["considered"], "lemmaSV": ["consider"], "LAT": [{"synset": "6202938", "text": "attitude", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6218486", "text": "position", "specificity": "0.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6217756", "text": "orientation", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "considered", "specificity": "0.0", "type": "LAT"}, {"text": "consider", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Paul the Apostle", "cookedLabel": "Paul the Apostle", "pageID": "24140", "editDist": 0.0, "labelProbability": 0.994799, "logPopularity": 5.762051382780177, "score": 0.975995274797593, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Copula (linguistics)", "cookedLabel": "Copula", "pageID": "5630", "editDist": 0.0, "labelProbability": 0.20202, "logPopularity": 3.5263605246161616, "score": 0.09310822918934004, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "To Be (Ayumi Hamasaki song)", "cookedLabel": "To Be", "pageID": "3427390", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.060443010546419, "score": 0.05168550302843927, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "apostle paul", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr000180", "qText": "which countries are part of the united kingdom?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7958392", "text": "people", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "countries", "specificity": "0.0", "type": "LAT"}, {"text": "country", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "High Sheriff of Lancashire", "cookedLabel": "High Sheriff of Lancashire", "pageID": "13807427", "editDist": 0.0, "labelProbability": 0.5, "logPopularity": 3.7376696182833684, "score": 0.31655453251418925, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "High Sheriff of Yorkshire", "cookedLabel": "High Sheriff of Yorkshire", "pageID": "7326096", "editDist": 0.0, "labelProbability": 0.5, "logPopularity": 3.6635616461296463, "score": 0.3070142417577131, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Art of the United Kingdom", "cookedLabel": "Art of the United Kingdom", "pageID": "1230235", "editDist": 1.0, "labelProbability": 0.0, "logPopularity": 3.871201010907891, "score": 0.15238646386916124, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "United Kingdom", "cookedLabel": "United Kingdom", "pageID": "31717", "editDist": 0.0, "labelProbability": 0.801464, "logPopularity": 11.570967932364097, "score": 0.9950615828111425, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the united kingdom?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000200", "qText": "what college did kevjumba?", "SV": ["kevjumba"], "lemmaSV": ["kevjumba"], "LAT": [{"synset": "8070328", "text": "institution", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8293263", "text": "educational institution", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7981699", "text": "body", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4348764", "text": "structure", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "2918337", "text": "building complex", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"text": "college", "specificity": "0.0", "type": "LAT"}], "Concept": [], "Clue": [] }, +{"qId": "wqr000220", "qText": "what sort of government does brazil have?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "13546128", "text": "operation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4749775", "text": "sameness", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13562370", "text": "processing", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4750845", "text": "similarity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "29976", "text": "process", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "13476660", "text": "data processing", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "sort", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Sort Of", "cookedLabel": "Sort Of", "pageID": "6111466", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.7376696182833684, "score": 0.007625638625247476, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Government", "cookedLabel": "Government", "pageID": "12229", "editDist": 0.0, "labelProbability": 0.139222, "logPopularity": 5.5093883366279774, "score": 0.15315169535105763, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Brazil", "cookedLabel": "Brazil", "pageID": "3383", "editDist": 0.0, "labelProbability": 0.671435, "logPopularity": 10.453572350254236, "score": 0.9909755240376303, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Brazil national football team", "cookedLabel": "Brazil national football team", "pageID": "149286", "editDist": 0.0, "labelProbability": 0.0639685, "logPopularity": 7.789868559054706, "score": 0.336803569077327, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "brazil", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000240", "qText": "what year was the great san francisco fire?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "15137796", "text": "time period", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7991473", "text": "gathering", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13597072", "text": "fundamental quantity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "year", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "List of people known as The Great", "cookedLabel": "the great", "pageID": "214273", "editDist": 0.0, "labelProbability": 0.392, "logPopularity": 2.5649493574615367, "score": 0.0899022043760746, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "1906 San Francisco earthquake", "cookedLabel": "1906 San Francisco earthquake", "pageID": "20110714", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.7612001156935624, "score": 0.926124312894242, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the great san francisco fire", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "great san francisco fire", "type": "CluePhrase", "weight": 0.99}, {"label": "san francisco fire", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000260", "qText": "where did rihanna grow up?", "SV": ["grow"], "lemmaSV": ["grow"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Rihanna", "cookedLabel": "Rihanna", "pageID": "2110323", "editDist": 0.0, "labelProbability": 0.990786, "logPopularity": 6.236369590203704, "score": 0.9743285979093527, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Grow Up (Svoy album)", "cookedLabel": "Grow Up", "pageID": "32182839", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.430816798843313, "score": 0.04643010512375539, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Grow Up, Tony Phillips", "cookedLabel": "Grow Up, Tony Phillips", "pageID": "41237889", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.2188758248682006, "score": 0.022990485979290737, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Grow Up (book)", "cookedLabel": "Grow Up", "pageID": "11645304", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.6635616461296463, "score": 0.02981111954944062, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Grow Up (The Queers album)", "cookedLabel": "Grow Up", "pageID": "8796937", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.9889840465642745, "score": 0.03600738939439352, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "grow up", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000280", "qText": "where is the kakadu national park located?", "SV": ["located"], "lemmaSV": ["locate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Kakadu National Park", "cookedLabel": "Kakadu National Park", "pageID": "101655", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.2626798770413155, "score": 0.9442496570627794, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000300", "qText": "where is the ottoman empire located?", "SV": ["located"], "lemmaSV": ["locate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Ottoman Empire", "cookedLabel": "Ottoman Empire", "pageID": "22278", "editDist": 0.0, "labelProbability": 0.968988, "logPopularity": 8.262300941787448, "score": 0.9938594960042121, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000320", "qText": "where is tom cruise from?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Tom Cruise", "cookedLabel": "Tom Cruise", "pageID": "31460", "editDist": 0.0, "labelProbability": 0.96695, "logPopularity": 5.442417710521793, "score": 0.9672458310948574, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000340", "qText": "what should you see in london?", "SV": ["see"], "lemmaSV": ["see"], "LAT": [{"text": "you", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "You", "cookedLabel": "You", "pageID": "464907", "editDist": 0.0, "labelProbability": 0.166744, "logPopularity": 3.332204510175204, "score": 0.01312126268186723, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "You (Ten Sharp song)", "cookedLabel": "You", "pageID": "18041571", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.454347296253507, "score": 0.007020891368638913, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "You (Juju album)", "cookedLabel": "You", "pageID": "32465927", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.584967478670572, "score": 0.0075889151861677365, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "To Know That You're Alive", "cookedLabel": "To Know That You're Alive", "pageID": "16113542", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.543294782270004, "score": 0.007402908081191978, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "You County", "cookedLabel": "You County", "pageID": "24702306", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.477336814478207, "score": 0.007117710644484163, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "B.B. King in London", "cookedLabel": "B.B. King in London", "pageID": "13253896", "editDist": 0.0, "labelProbability": 0.912892, "logPopularity": 3.970291913552122, "score": 0.7786155377852918, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "In London (Dewey Redman album)", "cookedLabel": "In London", "pageID": "31662165", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.8501476017100584, "score": 0.04583958488949033, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "In London (Ravi Shankar album)", "cookedLabel": "In London", "pageID": "23803159", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.007333185232471, "score": 0.05014577939757896, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "in london?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000360", "qText": "what did kate winslet get an oscar for?", "SV": ["get"], "lemmaSV": ["get"], "LAT": [{"text": "winslet", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Kate Winslet", "cookedLabel": "Kate Winslet", "pageID": "52707", "editDist": 0.0, "labelProbability": 0.998207, "logPopularity": 5.003946305945459, "score": 0.8624249110139923, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Academy Awards", "cookedLabel": "Academy Awards", "pageID": "324", "editDist": 0.0, "labelProbability": 0.680424, "logPopularity": 4.5217885770490405, "score": 0.30982989450411025, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Oscar De La Hoya", "cookedLabel": "Oscar De La Hoya", "pageID": "95310", "editDist": 0.0, "labelProbability": 0.096522, "logPopularity": 5.153291594497779, "score": 0.10716640977370843, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Oscar I of Sweden", "cookedLabel": "Oscar I of Sweden", "pageID": "38746", "editDist": 0.0, "labelProbability": 0.096522, "logPopularity": 5.062595033026967, "score": 0.10206990524289294, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Oscar II of Sweden", "cookedLabel": "Oscar II of Sweden", "pageID": "104650", "editDist": 0.0, "labelProbability": 0.096522, "logPopularity": 5.181783550292085, "score": 0.10881309435169641, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Oscar Wilde", "cookedLabel": "Oscar Wilde", "pageID": "22614", "editDist": 0.0, "labelProbability": 0.096522, "logPopularity": 5.605802066295998, "score": 0.13604722377665357, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "oscar", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr000380", "qText": "where did pavlova originate?", "SV": ["originate"], "lemmaSV": ["originate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Pavlova (food)", "cookedLabel": "Pavlova", "pageID": "67081", "editDist": 0.0, "labelProbability": 0.923677, "logPopularity": 3.5263605246161616, "score": 0.8457864866370466, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Anna Pavlova (gymnast)", "cookedLabel": "Anna Pavlova", "pageID": "1531928", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.343805421853684, "score": 0.11121386998928041, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Karolina Pavlova", "cookedLabel": "Karolina Pavlova", "pageID": "25434748", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.143134726391533, "score": 0.09985814500271337, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Anna Pavlova (film)", "cookedLabel": "Anna Pavlova", "pageID": "30828989", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.110873864173311, "score": 0.09813171286801613, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Anna Pavlova", "cookedLabel": "Anna Pavlova", "pageID": "63157", "editDist": 0.0, "labelProbability": 0.0607112, "logPopularity": 4.605170185988092, "score": 0.16556105820547126, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "pavlova", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000400", "qText": "what state is harvard college located?", "SV": ["located"], "lemmaSV": ["locate"], "LAT": [{"synset": "7495208", "text": "emotion", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8139116", "text": "federal department", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8094128", "text": "administrative unit", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "11428673", "text": "natural phenomenon", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "34512", "text": "phenomenon", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8136796", "text": "government department", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8140150", "text": "executive department", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8067137", "text": "polity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14009303", "text": "emotional state", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "11429173", "text": "chemical phenomenon", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8067430", "text": "government", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8237635", "text": "division", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "26390", "text": "feeling", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8131836", "text": "department", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "state", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Harvard College", "cookedLabel": "Harvard College", "pageID": "260879", "editDist": 0.0, "labelProbability": 0.884069, "logPopularity": 6.642486801367256, "score": 0.9764428806488475, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Harvard University", "cookedLabel": "Harvard University", "pageID": "18426501", "editDist": 0.0, "labelProbability": 0.113872, "logPopularity": 8.537191877922927, "score": 0.5830172367337914, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "harvard college", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr000420", "qText": "what state did al gore represent?", "SV": ["represent"], "lemmaSV": ["represent"], "LAT": [{"synset": "7495208", "text": "emotion", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8139116", "text": "federal department", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8094128", "text": "administrative unit", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "11428673", "text": "natural phenomenon", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "34512", "text": "phenomenon", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8136796", "text": "government department", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8140150", "text": "executive department", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8067137", "text": "polity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14009303", "text": "emotional state", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "11429173", "text": "chemical phenomenon", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8067430", "text": "government", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8237635", "text": "division", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "26390", "text": "feeling", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8131836", "text": "department", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "state", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Al Gore", "cookedLabel": "Al Gore", "pageID": "5042706", "editDist": 0.0, "labelProbability": 0.973792, "logPopularity": 6.0844994130751715, "score": 0.9781639206533114, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000440", "qText": "what was the first name of the washington redskins?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9659294", "text": "person of color", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9664887", "text": "Amerindian", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "redskins", "specificity": "0.0", "type": "LAT"}, {"text": "redskin", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Given name", "cookedLabel": "Given name", "pageID": "247991", "editDist": 0.0, "labelProbability": 0.666667, "logPopularity": 3.4965075614664802, "score": 0.6979694032215839, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Solomon Islands", "cookedLabel": "Solomon Islands", "pageID": "265083", "editDist": 0.0, "labelProbability": 0.333333, "logPopularity": 6.6895992691789665, "score": 0.5585770570841422, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Louis IX of France", "cookedLabel": "Louis IX of France", "pageID": "18549", "editDist": 0.0, "labelProbability": 0.333333, "logPopularity": 5.241747015059643, "score": 0.3467612192375154, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Division of Solomon", "cookedLabel": "Division of Solomon", "pageID": "2171828", "editDist": 0.0, "labelProbability": 0.333333, "logPopularity": 5.117993812416755, "score": 0.3301389058568271, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Solomon Burke", "cookedLabel": "Solomon Burke", "pageID": "164477", "editDist": 0.0, "labelProbability": 0.333333, "logPopularity": 5.262690188904886, "score": 0.3496130227702334, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Washington Redskins", "cookedLabel": "Washington Redskins", "pageID": "33673", "editDist": 0.0, "labelProbability": 0.883442, "logPopularity": 8.042056410058754, "score": 0.8942421484366897, "getByLAT": 1, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the first name of the washington redskins", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "first name of the washington redskins", "type": "CluePhrase", "weight": 0.99}, {"label": "the washington redskins?", "type": "ClueNE", "weight": 1.6}] }, +{"qId": "wqr000460", "qText": "what made ancient rome fall?", "SV": ["fall"], "lemmaSV": ["fall"], "LAT": [{"synset": "8540894", "text": "center", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "8693705", "text": "urban area", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8665520", "text": "seat", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7981699", "text": "body", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8398167", "text": "leadership", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8542298", "text": "city", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8709407", "text": "national capital", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "8514304", "text": "area", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8643858", "text": "municipality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8535783", "text": "capital", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "rome", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Ancient Rome", "cookedLabel": "Ancient Rome", "pageID": "521555", "editDist": 0.0, "labelProbability": 0.872097, "logPopularity": 5.332718793265369, "score": 0.8104863546947404, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000480", "qText": "who does kris humphries play for in the nba?", "SV": ["play"], "lemmaSV": ["play"], "LAT": [{"synset": "38116", "text": "action", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "41926", "text": "playing", "specificity": "0.0", "type": "WordnetLAT"}, {"text": "play", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Kris Humphries", "cookedLabel": "Kris Humphries", "pageID": "2312705", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.51085950651685, "score": 0.9515892730037312, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "National Basketball Association", "cookedLabel": "National Basketball Association", "pageID": "22093", "editDist": 0.0, "labelProbability": 0.881188, "logPopularity": 7.202661196523238, "score": 0.9379815949530763, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "nba", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr000500", "qText": "where did sir ernest shackleton come from?", "SV": ["come"], "lemmaSV": ["come"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Ernest Shackleton", "cookedLabel": "Ernest Shackleton", "pageID": "60004", "editDist": 0.0, "labelProbability": 0.994582, "logPopularity": 4.5217885770490405, "score": 0.9507370650288589, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "COMEFROM", "cookedLabel": "COMEFROM", "pageID": "994284", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 1.6094379124341003, "score": 0.012369376838424827, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "sir ernest shackleton", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr000520", "qText": "who does the islamic worship?", "SV": ["does"], "lemmaSV": ["do"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Islam", "cookedLabel": "Islam", "pageID": "6037917", "editDist": 0.0, "labelProbability": 0.714034, "logPopularity": 8.91811465947453, "score": 0.9515347415314166, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "islamic worship", "type": "CluePhrase", "weight": 0.99}, {"label": "the islamic", "type": "ClueNE", "weight": 1.11}, {"label": "worship", "type": "ClueToken", "weight": 1.0}] }, +{"qId": "wqr000540", "qText": "what does pixar produce?", "SV": ["produce"], "lemmaSV": ["produce"], "LAT": [{"text": "pixar", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Pixar", "cookedLabel": "Pixar", "pageID": "78969", "editDist": 0.0, "labelProbability": 0.982097, "logPopularity": 5.351858133476067, "score": 0.8368938289140296, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Question mark", "cookedLabel": "Question mark", "pageID": "59348", "editDist": 0.0, "labelProbability": 0.877681, "logPopularity": 5.262690188904886, "score": 0.8229142522615168, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "?", "type": "ClueNE", "weight": 2.1}] }, +{"qId": "wqr000560", "qText": "how many teams are there in the ncaa football?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33914", "text": "amount", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "How Many", "cookedLabel": "How Many", "pageID": "10680822", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.7612001156935624, "score": 0.7680939451144396, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "College football", "cookedLabel": "College football", "pageID": "6771", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 6.137727054086234, "score": 0.15933808868098598, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "teams", "type": "ClueToken", "weight": 1.0}, {"label": "the ncaa football?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000580", "qText": "what movies has taylor lautner?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "4014270", "text": "product", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6631572", "text": "show", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3133774", "text": "creation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7303344", "text": "social event", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "movies", "specificity": "0.0", "type": "LAT"}, {"text": "movie", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Taylor Lautner", "cookedLabel": "Taylor Lautner", "pageID": "13199916", "editDist": 0.0, "labelProbability": 0.999293, "logPopularity": 4.700480365792417, "score": 0.8902076395507517, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000600", "qText": "what episode does rukia fade away?", "SV": ["fade"], "lemmaSV": ["fade"], "LAT": [{"synset": "6360590", "text": "written communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7303344", "text": "social event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "3932650", "text": "photographic paper", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6403644", "text": "section", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6631572", "text": "show", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6631935", "text": "broadcast", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3580409", "text": "instrumentality", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7298313", "text": "happening", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "3932386", "text": "photographic equipment", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6374360", "text": "writing", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3298959", "text": "equipment", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "3343766", "text": "film", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "episode", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Rukia Kuchiki", "cookedLabel": "Rukia Kuchiki", "pageID": "2356421", "editDist": 0.0, "labelProbability": 0.792359, "logPopularity": 4.143134726391533, "score": 0.8128115471721565, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Rukia (bird)", "cookedLabel": "Rukia", "pageID": "13050373", "editDist": 0.0, "labelProbability": 0.0963455, "logPopularity": 3.6109179126442243, "score": 0.11403551989252897, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Fade Away (song)", "cookedLabel": "Fade Away", "pageID": "16757390", "editDist": 0.0, "labelProbability": 0.561111, "logPopularity": 4.060443010546419, "score": 0.42427332414050917, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Cigarettes & Alcohol", "cookedLabel": "Cigarettes & Alcohol", "pageID": "1460994", "editDist": 0.0, "labelProbability": 0.0777778, "logPopularity": 4.30406509320417, "score": 0.08465432110351759, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Another Animal", "cookedLabel": "Another Animal", "pageID": "13057317", "editDist": 0.0, "labelProbability": 0.0777778, "logPopularity": 4.574710978503383, "score": 0.09811550264895015, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Fade Away (EP)", "cookedLabel": "Fade Away", "pageID": "26755756", "editDist": 0.0, "labelProbability": 0.0777778, "logPopularity": 3.6109179126442243, "score": 0.057507839991317325, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Fade Away (Best Coast album)", "cookedLabel": "Fade Away", "pageID": "40558516", "editDist": 0.0, "labelProbability": 0.0777778, "logPopularity": 4.1588830833596715, "score": 0.07814418086189585, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "rukia", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000620", "qText": "who does kurt busch drive for now?", "SV": ["drive"], "lemmaSV": ["drive"], "LAT": [{"text": "busch", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Kurt Busch", "cookedLabel": "Kurt Busch", "pageID": "525736", "editDist": 0.0, "labelProbability": 0.99964, "logPopularity": 5.081404364984463, "score": 0.8686002704395099, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Jordin Sparks discography", "cookedLabel": "Jordin Sparks discography", "pageID": "18194927", "editDist": 0.0, "labelProbability": 0.867925, "logPopularity": 3.4657359027997265, "score": 0.601860125368496, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "for now", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000640", "qText": "what team is chris paul on?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "7957410", "text": "biological group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8010371", "text": "animal group", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "team", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Chris Paul", "cookedLabel": "Chris Paul", "pageID": "4987149", "editDist": 0.0, "labelProbability": 0.999753, "logPopularity": 4.718498871295094, "score": 0.9569693241599244, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000660", "qText": "what happened to justin bieber 2012?", "SV": ["happened"], "lemmaSV": ["happen"], "LAT": [], "Concept": [{"fullLabel": "What Happened", "cookedLabel": "What Happened", "pageID": "17648735", "editDist": 0.0, "labelProbability": 0.950207, "logPopularity": 3.970291913552122, "score": 0.7491640697900563, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Justin Bieber", "cookedLabel": "Justin Bieber", "pageID": "23680998", "editDist": 0.0, "labelProbability": 0.995669, "logPopularity": 5.991464547107982, "score": 0.925242503944315, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "2012", "cookedLabel": "2012", "pageID": "47374", "editDist": 0.0, "labelProbability": 0.370218, "logPopularity": 2.833213344056216, "score": 0.0950054105606654, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "2012 phenomenon", "cookedLabel": "2012 phenomenon", "pageID": "21538638", "editDist": 0.0, "labelProbability": 0.062167, "logPopularity": 4.219507705176107, "score": 0.02137140937577736, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "2012 (film)", "cookedLabel": "2012", "pageID": "18436536", "editDist": 0.0, "labelProbability": 0.128225, "logPopularity": 4.736198448394496, "score": 0.03877421682047329, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "justin bieber 2012", "type": "CluePhrase", "weight": 0.99}, {"label": "2012", "type": "ClueNE", "weight": 2.1}] }, +{"qId": "wqr000680", "qText": "where did kurds originate from?", "SV": ["originate"], "lemmaSV": ["originate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Kurdish people", "cookedLabel": "Kurdish people", "pageID": "17068", "editDist": 0.0, "labelProbability": 0.91787, "logPopularity": 5.25227342804663, "score": 0.9376589245135472, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "kurds", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000700", "qText": "who played juni in spy kids 4?", "SV": ["played"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "June", "cookedLabel": "June", "pageID": "15785", "editDist": 0.0, "labelProbability": 0.957049, "logPopularity": 2.8903717578961645, "score": 0.3756061986134487, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Juhni", "cookedLabel": "Juhni", "pageID": "40864001", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.060443010546419, "score": 0.03752557266374662, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Juni (album)", "cookedLabel": "Juni", "pageID": "33741916", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.828641396489095, "score": 0.03281312068885659, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Juan de Juni", "cookedLabel": "Juan de Juni", "pageID": "14349994", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.5553480614894135, "score": 0.027989524351936867, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Juni Cortez", "cookedLabel": "Juni Cortez", "pageID": "565107", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.912023005428146, "score": 0.03443848150516163, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Spy Kids: All the Time in the World", "cookedLabel": "Spy Kids: All the Time in the World", "pageID": "29384326", "editDist": 0.0, "labelProbability": 0.942177, "logPopularity": 4.174387269895637, "score": 0.8197537507990824, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "spy kids 4", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000720", "qText": "what are the major imports of the united states?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9652940", "text": "traveler", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10123254", "text": "foreigner", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5145753", "text": "value", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3080712", "text": "commodity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5145473", "text": "worth", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5177340", "text": "significance", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6611268", "text": "message", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5928460", "text": "meaning", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5175788", "text": "importance", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "imports", "specificity": "0.0", "type": "LAT"}, {"text": "import", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "International trade", "cookedLabel": "International trade", "pageID": "14567", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.356708826689592, "score": 0.5335993809709476, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Water export", "cookedLabel": "Water export", "pageID": "12517724", "editDist": 3.0, "labelProbability": 0.0, "logPopularity": 1.6094379124341003, "score": 0.014535818203352374, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Major", "cookedLabel": "Major", "pageID": "201920", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 6.82001636467413, "score": 0.16955685053201597, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "The Major", "cookedLabel": "The Major", "pageID": "9600545", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.995732273553991, "score": 0.020167690730618193, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "United States", "cookedLabel": "United States", "pageID": "3434750", "editDist": 0.0, "labelProbability": 0.836336, "logPopularity": 13.02522232257073, "score": 0.9982363165377705, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "States and union territories of India", "cookedLabel": "States and union territories of India", "pageID": "375986", "editDist": 0.0, "labelProbability": 0.0869492, "logPopularity": 9.911654115202522, "score": 0.6661774336773838, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "States of Nigeria", "cookedLabel": "States of Nigeria", "pageID": "226734", "editDist": 0.0, "labelProbability": 0.0869492, "logPopularity": 6.932447891572509, "score": 0.2503879062273689, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Administrative divisions of Mexico", "cookedLabel": "Administrative divisions of Mexico", "pageID": "87990", "editDist": 0.0, "labelProbability": 0.0869492, "logPopularity": 7.814803429489359, "score": 0.36189680695012594, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "States of Brazil", "cookedLabel": "States of Brazil", "pageID": "229379", "editDist": 0.0, "labelProbability": 0.165626, "logPopularity": 8.14902386805177, "score": 0.49875640020650774, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "U.S. state", "cookedLabel": "U.S. state", "pageID": "18618239", "editDist": 0.0, "labelProbability": 0.198042, "logPopularity": 8.919453168575453, "score": 0.6470905751810023, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the major imports of the united states", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "major imports of the united states", "type": "CluePhrase", "weight": 0.99}, {"label": "states?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000740", "qText": "where did martin luther king junior go to college?", "SV": ["go"], "lemmaSV": ["go"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Martin Luther King, Jr.", "cookedLabel": "Martin Luther King, Jr.", "pageID": "20076", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 5.786897381366708, "score": 0.9768880164204938, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Goto", "cookedLabel": "Goto", "pageID": "23307350", "editDist": 0.0, "labelProbability": 0.233129, "logPopularity": 3.258096538021482, "score": 0.06728267116099904, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "College", "cookedLabel": "College", "pageID": "5689", "editDist": 0.0, "labelProbability": 0.29875, "logPopularity": 5.497168225293202, "score": 0.27206828588388454, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lists of American institutions of higher education", "cookedLabel": "Lists of American institutions of higher education", "pageID": "322811", "editDist": 0.0, "labelProbability": 0.0517783, "logPopularity": 2.8903717578961645, "score": 0.009291532739167302, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "List of college athletic conferences in the United States", "cookedLabel": "college", "pageID": "577952", "editDist": 0.0, "labelProbability": 0.304197, "logPopularity": 1.9459101490553132, "score": 0.016696137164309035, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "martin luther king junior", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr000760", "qText": "what legal system does germany use?", "SV": ["use"], "lemmaSV": ["use"], "LAT": [{"synset": "19308", "text": "natural object", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5226062", "text": "live body", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5227735", "text": "body part", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5223633", "text": "body", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9408804", "text": "part", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "3580409", "text": "instrumentality", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5911139", "text": "plan of action", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5624569", "text": "know-how", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5624029", "text": "ability", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "2452", "text": "thing", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5907175", "text": "plan", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5734290", "text": "structure", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5668113", "text": "method", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4774586", "text": "regularity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "21007", "text": "matter", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4775722", "text": "orderliness", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "system", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "List of national legal systems", "cookedLabel": "legal system", "pageID": "154708", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.833213344056216, "score": 0.004446205345094004, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Germany", "cookedLabel": "Germany", "pageID": "11867", "editDist": 0.0, "labelProbability": 0.731908, "logPopularity": 11.210644004861829, "score": 0.995640149842911, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000780", "qText": "who invented arabic alphabet?", "SV": ["invented"], "lemmaSV": ["invent"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Arabic alphabet", "cookedLabel": "Arabic alphabet", "pageID": "2204", "editDist": 0.0, "labelProbability": 0.782609, "logPopularity": 4.7535901911063645, "score": 0.7556064975707828, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Morse code for non-Latin alphabets", "cookedLabel": "Morse code for non-Latin alphabets", "pageID": "17878606", "editDist": 0.0, "labelProbability": 0.211957, "logPopularity": 2.5649493574615367, "score": 0.02202449782540842, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "arabic alphabet", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr000800", "qText": "where is lake waynoka ohio?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Lake Waynoka, Ohio", "cookedLabel": "Lake Waynoka, Ohio", "pageID": "9176251", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.356708826689592, "score": 0.8256172039061515, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "lake waynoka ohio", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "lake waynoka", "type": "ClueNE", "weight": 1.11}, {"label": "ohio", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr000820", "qText": "where is made kia car?", "SV": ["made"], "lemmaSV": ["make"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Skip car", "cookedLabel": "Skip car", "pageID": "22258640", "editDist": 2.0, "labelProbability": 0.0, "logPopularity": 0.6931471805599453, "score": 0.009538173494138642, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "kia car", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr000840", "qText": "what is the zip code for moorpark ca?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6371284", "text": "writing", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6365164", "text": "coding system", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6360590", "text": "written communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "code", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "ZIP code", "cookedLabel": "ZIP code", "pageID": "51550", "editDist": 0.0, "labelProbability": 0.965969, "logPopularity": 10.142898076314209, "score": 0.9915968566201331, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Genetics", "cookedLabel": "Genetics", "pageID": "12266", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 5.991464547107982, "score": 0.9266078744173097, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Moorpark, California", "cookedLabel": "Moorpark, California", "pageID": "108333", "editDist": 0.0, "labelProbability": 0.94247, "logPopularity": 5.308267697401205, "score": 0.9457017672563924, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the zip code for moorpark ca", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "zip code for moorpark ca", "type": "CluePhrase", "weight": 0.99}, {"label": "moorpark", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000860", "qText": "what other cars does gm make?", "SV": ["make"], "lemmaSV": ["make"], "LAT": [{"synset": "3099154", "text": "container", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "3796768", "text": "motor vehicle", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4112987", "text": "room", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3580409", "text": "instrumentality", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "4531608", "text": "vehicle", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4348764", "text": "structure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "2738693", "text": "area", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "3083745", "text": "compartment", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3105141", "text": "conveyance", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "4177098", "text": "self-propelled vehicle", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4583497", "text": "wheeled vehicle", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "cars", "specificity": "0.0", "type": "LAT"}, {"text": "car", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Germany", "cookedLabel": "Germany", "pageID": "11867", "editDist": 0.0, "labelProbability": 0.0509596, "logPopularity": 11.210644004861829, "score": 0.9089572500239089, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "List of Latin-script digraphs", "cookedLabel": "gm", "pageID": "22469831", "editDist": 0.0, "labelProbability": 0.0509596, "logPopularity": 6.22455842927536, "score": 0.3338934725624929, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "General Mills", "cookedLabel": "General Mills", "pageID": "164902", "editDist": 0.0, "labelProbability": 0.0509596, "logPopularity": 4.875197323201151, "score": 0.18239130735317602, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "General manager", "cookedLabel": "General manager", "pageID": "627189", "editDist": 0.0, "labelProbability": 0.0509596, "logPopularity": 5.214935757608986, "score": 0.21477318267976536, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "General Motors", "cookedLabel": "General Motors", "pageID": "12102", "editDist": 0.0, "labelProbability": 0.0509596, "logPopularity": 6.863803391452954, "score": 0.42382576537505606, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "other", "type": "ClueToken", "weight": 1.0}, {"label": "gm", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000880", "qText": "what did richard nixon do for a living before he became president?", "SV": ["do"], "lemmaSV": ["do"], "LAT": [{"synset": "9633690", "text": "communicator", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "10486961", "text": "President of the United States", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10371605", "text": "negotiator", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10184340", "text": "head of state", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "10541628", "text": "representative", "specificity": "-3.0", "type": "WordnetLAT"}, {"text": "nixon", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Richard Nixon", "cookedLabel": "Richard Nixon", "pageID": "25473", "editDist": 0.0, "labelProbability": 0.987793, "logPopularity": 6.630683385642372, "score": 0.9406843589036394, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "For a Living", "cookedLabel": "For a Living", "pageID": "14785803", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.8066624897703196, "score": 0.7729170949611157, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Helium", "cookedLabel": "Helium", "pageID": "13256", "editDist": 0.0, "labelProbability": 0.520629, "logPopularity": 3.8066624897703196, "score": 0.1229830065885628, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "He", "cookedLabel": "He", "pageID": "225073", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.772588722239781, "score": 0.0176852074788872, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "President", "cookedLabel": "President", "pageID": "24110", "editDist": 0.0, "labelProbability": 0.153158, "logPopularity": 7.502186486602924, "score": 0.3892706865031215, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "President of the United States", "cookedLabel": "President of the United States", "pageID": "24113", "editDist": 0.0, "labelProbability": 0.280447, "logPopularity": 6.082218910376446, "score": 0.15405196892621265, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "president", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr000900", "qText": "where did henry hudson travel?", "SV": ["travel"], "lemmaSV": ["travel"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Henry Hudson", "cookedLabel": "Henry Hudson", "pageID": "44014", "editDist": 0.0, "labelProbability": 0.97447, "logPopularity": 4.189654742026425, "score": 0.9351326584066884, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000920", "qText": "what did egyptians speak?", "SV": ["speak"], "lemmaSV": ["speak"], "LAT": [{"synset": "6916947", "text": "natural language", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6293304", "text": "language", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9657682", "text": "African", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6999218", "text": "Afroasiatic", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "egyptians", "specificity": "0.0", "type": "LAT"}, {"text": "egyptian", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Egyptians", "cookedLabel": "Egyptians", "pageID": "31912046", "editDist": 0.0, "labelProbability": 0.481658, "logPopularity": 5.84354441703136, "score": 0.40854932616940715, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Ancient Egypt", "cookedLabel": "Ancient Egypt", "pageID": "874", "editDist": 0.0, "labelProbability": 0.237587, "logPopularity": 4.836281906951478, "score": 0.043851998974517345, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Egypt", "cookedLabel": "Egypt", "pageID": "8087628", "editDist": 0.0, "labelProbability": 0.127367, "logPopularity": 8.766394277049736, "score": 0.22605990409483226, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "egyptians", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr000940", "qText": "where did barack obama attend school?", "SV": ["attend"], "lemmaSV": ["attend"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Barack Obama", "cookedLabel": "Barack Obama", "pageID": "534366", "editDist": 0.0, "labelProbability": 0.987254, "logPopularity": 7.902487437162855, "score": 0.9929996109483711, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "School", "cookedLabel": "School", "pageID": "28022", "editDist": 0.0, "labelProbability": 0.351585, "logPopularity": 4.634728988229636, "score": 0.2211887797516702, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "School psychology", "cookedLabel": "School psychology", "pageID": "466785", "editDist": 0.0, "labelProbability": 0.0933467, "logPopularity": 2.772588722239781, "score": 0.010467927085618627, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "List of schools of philosophy", "cookedLabel": "school", "pageID": "7950118", "editDist": 0.0, "labelProbability": 0.274652, "logPopularity": 2.3978952727983707, "score": 0.019070493600576166, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "school", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr000960", "qText": "where north dakota located?", "SV": ["north"], "lemmaSV": ["north"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "North Dakota", "cookedLabel": "North Dakota", "pageID": "21651", "editDist": 0.0, "labelProbability": 0.623677, "logPopularity": 7.631916513071252, "score": 0.941929207138752, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr000980", "qText": "who played todd manning on one life to live?", "SV": ["played"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Todd Manning", "cookedLabel": "Todd Manning", "pageID": "713342", "editDist": 0.0, "labelProbability": 0.997139, "logPopularity": 4.23410650459726, "score": 0.8127787319459592, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "On One", "cookedLabel": "On One", "pageID": "18893398", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.713572066704308, "score": 0.7629648266763546, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "One Life to Live", "cookedLabel": "One Life to Live", "pageID": "341990", "editDist": 0.0, "labelProbability": 0.917899, "logPopularity": 5.517452896464707, "score": 0.9010468545616608, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001000", "qText": "where do the abenaki indians live?", "SV": ["live"], "lemmaSV": ["live"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Abenaki", "cookedLabel": "Abenaki", "pageID": "55012", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.970291913552122, "score": 0.9342618591720794, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "abenaki indians", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr001020", "qText": "where is isthmus of panama located?", "SV": ["located"], "lemmaSV": ["locate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Isthmus of Panama", "cookedLabel": "Isthmus of Panama", "pageID": "1404472", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.4965075614664802, "score": 0.9144961673270963, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001040", "qText": "what states does the connecticut river flow through?", "SV": ["flow"], "lemmaSV": ["flow"], "LAT": [{"synset": "7495208", "text": "emotion", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8139116", "text": "federal department", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8094128", "text": "administrative unit", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "11428673", "text": "natural phenomenon", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "34512", "text": "phenomenon", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8136796", "text": "government department", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8140150", "text": "executive department", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8067137", "text": "polity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14009303", "text": "emotional state", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "11429173", "text": "chemical phenomenon", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8067430", "text": "government", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8237635", "text": "division", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "26390", "text": "feeling", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8131836", "text": "department", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "states", "specificity": "0.0", "type": "LAT"}, {"text": "state", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Connecticut River", "cookedLabel": "Connecticut River", "pageID": "252145", "editDist": 0.0, "labelProbability": 0.997329, "logPopularity": 5.802118375377063, "score": 0.9768169065974456, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "River", "cookedLabel": "River", "pageID": "18842395", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 5.117993812416755, "score": 0.06849977248051861, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Preposition and postposition", "cookedLabel": "Preposition and postposition", "pageID": "199358", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.6390573296152584, "score": 0.016345844972803406, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "river flow", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001060", "qText": "who is susan st james?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "9434308", "text": "river", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9818234", "text": "Apostle", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10184340", "text": "head of state", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "9248053", "text": "body of water", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "2452", "text": "thing", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9651570", "text": "religious person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10371605", "text": "negotiator", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "10035803", "text": "disciple", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9644715", "text": "intellectual", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9997190", "text": "criminal", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10251212", "text": "king", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10560786", "text": "ruler", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9657157", "text": "wrongdoer", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6454286", "text": "Epistle", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10566702", "text": "saint", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10119144", "text": "follower", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9528550", "text": "deity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10648006", "text": "sovereign", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "10813654", "text": "writer", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9697405", "text": "Christian", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9527267", "text": "spiritual being", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10577282", "text": "scholar", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9471510", "text": "stream", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9633690", "text": "communicator", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5950141", "text": "belief", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10443334", "text": "philosopher", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10508450", "text": "psychologist", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10580065", "text": "scientist", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9851208", "text": "bad person", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "6374360", "text": "writing", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10541628", "text": "representative", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10494535", "text": "principal", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10253142", "text": "King of England", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6403644", "text": "section", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6406508", "text": "book", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6360590", "text": "written communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"text": "james", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Whois", "cookedLabel": "Whois", "pageID": "4315433", "editDist": 0.0, "labelProbability": 0.0673077, "logPopularity": 3.1780538303479458, "score": 0.031085894047283118, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Susan Saint James", "cookedLabel": "Susan Saint James", "pageID": "937621", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.574710978503383, "score": 0.04585533570323418, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Susan Saint James", "cookedLabel": "Susan Saint James", "pageID": "937621", "editDist": 0.0, "labelProbability": 0.227561, "logPopularity": 4.574710978503383, "score": 0.13414007012758933, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Saint James School (Montgomery, Alabama)", "cookedLabel": "Saint James School", "pageID": "4654822", "editDist": 0.0, "labelProbability": 0.227561, "logPopularity": 4.30406509320417, "score": 0.11637416465831421, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Saint James, Indiana", "cookedLabel": "Saint James, Indiana", "pageID": "15124957", "editDist": 0.0, "labelProbability": 0.227561, "logPopularity": 4.0943445622221, "score": 0.10404626095875653, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Saint James, Barbados", "cookedLabel": "Saint James, Barbados", "pageID": "1528028", "editDist": 0.0, "labelProbability": 0.227561, "logPopularity": 4.442651256490317, "score": 0.1252010789745587, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Saint James Parish, New Brunswick", "cookedLabel": "Saint James Parish, New Brunswick", "pageID": "39041822", "editDist": 0.0, "labelProbability": 0.227561, "logPopularity": 4.477336814478207, "score": 0.12749824947416874, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "susan st james", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr001080", "qText": "what cancer did audrey hepburn died of?", "SV": ["died"], "lemmaSV": ["die"], "LAT": [{"synset": "7957410", "text": "biological group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8008892", "text": "taxonomic group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "14262907", "text": "malignant tumor", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14075528", "text": "ill health", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8703415", "text": "sign of the zodiac", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14075399", "text": "pathological state", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "14085287", "text": "illness", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13943868", "text": "condition", "specificity": "-8.0", "type": "WordnetLAT"}, {"synset": "14258682", "text": "tumor", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "1765166", "text": "arthropod genus", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "24900", "text": "state", "specificity": "-9.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14093842", "text": "disease", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "14057659", "text": "physical condition", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "8125938", "text": "genus", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9275876", "text": "constellation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8647614", "text": "region", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "14261043", "text": "malignancy", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "14257556", "text": "growth", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "19308", "text": "natural object", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "cancer", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Audrey Hepburn", "cookedLabel": "Audrey Hepburn", "pageID": "52139", "editDist": 0.0, "labelProbability": 0.99483, "logPopularity": 5.1647859739235145, "score": 0.9660063566164164, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001100", "qText": "what do the buddha believe in?", "SV": ["believe"], "lemmaSV": ["believe"], "LAT": [{"synset": "10363285", "text": "mystic", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10158287", "text": "good person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9867135", "text": "believer", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9651570", "text": "religious person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10566407", "text": "saint", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "buddha", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Gautama Buddha", "cookedLabel": "Gautama Buddha", "pageID": "3395", "editDist": 0.0, "labelProbability": 0.521722, "logPopularity": 5.420534999272286, "score": 0.47386472811390146, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Buddhahood", "cookedLabel": "Buddhahood", "pageID": "174976", "editDist": 0.0, "labelProbability": 0.313969, "logPopularity": 3.4657359027997265, "score": 0.03848139367395044, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Belief", "cookedLabel": "Belief", "pageID": "102883", "editDist": 0.0, "labelProbability": 0.928571, "logPopularity": 2.70805020110221, "score": 0.559067381405877, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "believe in", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001120", "qText": "who became president after harding died?", "SV": ["became"], "lemmaSV": ["become"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "President", "cookedLabel": "President", "pageID": "24110", "editDist": 0.0, "labelProbability": 0.153158, "logPopularity": 7.502186486602924, "score": 0.3892706865031215, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "President of the United States", "cookedLabel": "President of the United States", "pageID": "24113", "editDist": 0.0, "labelProbability": 0.280447, "logPopularity": 6.082218910376446, "score": 0.15405196892621265, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Harding Township, New Jersey", "cookedLabel": "Harding Township, New Jersey", "pageID": "125546", "editDist": 0.0, "labelProbability": 0.211929, "logPopularity": 4.955827057601261, "score": 0.3291666404956828, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Harding, Minnesota", "cookedLabel": "Harding, Minnesota", "pageID": "120849", "editDist": 0.0, "labelProbability": 0.211929, "logPopularity": 4.584967478670572, "score": 0.28201907346285054, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Harding, Wisconsin", "cookedLabel": "Harding, Wisconsin", "pageID": "139292", "editDist": 0.0, "labelProbability": 0.211929, "logPopularity": 4.564348191467836, "score": 0.2795208389287822, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Harding County, South Dakota", "cookedLabel": "Harding County, South Dakota", "pageID": "91813", "editDist": 0.0, "labelProbability": 0.211929, "logPopularity": 4.605170185988092, "score": 0.2844799393380819, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Warren G. Harding", "cookedLabel": "Warren G. Harding", "pageID": "33060", "editDist": 0.0, "labelProbability": 0.290018, "logPopularity": 5.720311776607412, "score": 0.2931267792954646, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Death", "cookedLabel": "Death", "pageID": "8221", "editDist": 0.0, "labelProbability": 0.32493, "logPopularity": 4.90527477843843, "score": 0.22812175118227904, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "president after harding died", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "harding", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001140", "qText": "who was the leader of germany in wwii?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "3754377", "text": "merchandise", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3330714", "text": "feature", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3080712", "text": "commodity", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "leader", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "President of Germany (1919–45)", "cookedLabel": "President of Germany", "pageID": "407083", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.6888794541139363, "score": 0.4338672682742019, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "States of Germany", "cookedLabel": "States of Germany", "pageID": "217450", "editDist": 2.0, "labelProbability": 0.0, "logPopularity": 4.727387818712341, "score": 0.06557484871615457, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Leadership", "cookedLabel": "Leadership", "pageID": "130918", "editDist": 0.0, "labelProbability": 0.303262, "logPopularity": 4.343805421853684, "score": 0.21074447522291953, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Leader of the Opposition (United Kingdom)", "cookedLabel": "Leader of the Opposition", "pageID": "711239", "editDist": 0.0, "labelProbability": 0.0698267, "logPopularity": 4.663439094112067, "score": 0.03963653931791342, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Leader (comics)", "cookedLabel": "Leader", "pageID": "1584994", "editDist": 0.0, "labelProbability": 0.151631, "logPopularity": 4.060443010546419, "score": 0.0401814825039581, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Germany", "cookedLabel": "Germany", "pageID": "11867", "editDist": 0.0, "labelProbability": 0.731908, "logPopularity": 11.210644004861829, "score": 0.9882824313361198, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "World War II", "cookedLabel": "World War II", "pageID": "32927", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 10.052123051675276, "score": 0.5867284583132142, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "wwii", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr001160", "qText": "when did george w bush take office?", "SV": ["take"], "lemmaSV": ["take"], "LAT": [{"synset": "15147173", "text": "time", "specificity": "0.0", "type": "QuestionWordLAT"}, {"synset": "15184543", "text": "date", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "George W. Bush", "cookedLabel": "George W. Bush", "pageID": "3414021", "editDist": 0.0, "labelProbability": 0.976972, "logPopularity": 7.558516743045645, "score": 0.9909967416898372, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Office", "cookedLabel": "Office", "pageID": "382507", "editDist": 0.0, "labelProbability": 0.315404, "logPopularity": 5.53338948872752, "score": 0.291962500202855, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Office (UK TV series)", "cookedLabel": "The Office", "pageID": "2995581", "editDist": 0.0, "labelProbability": 0.0649595, "logPopularity": 4.948759890378168, "score": 0.03312697862285286, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "OpenOffice.org", "cookedLabel": "OpenOffice.org", "pageID": "68227", "editDist": 0.0, "labelProbability": 0.0649595, "logPopularity": 5.075173815233827, "score": 0.03564425763965933, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "The Office (U.S. TV series)", "cookedLabel": "The Office", "pageID": "2995553", "editDist": 0.0, "labelProbability": 0.0649595, "logPopularity": 6.253828811575473, "score": 0.06973986455768874, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Microsoft Office", "cookedLabel": "Microsoft Office", "pageID": "20288", "editDist": 0.0, "labelProbability": 0.384022, "logPopularity": 4.672828834461906, "score": 0.11177829516220708, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "george w bush", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr001180", "qText": "where is wellsville missouri?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Missouri", "cookedLabel": "Missouri", "pageID": "19571", "editDist": 0.0, "labelProbability": 0.726548, "logPopularity": 8.925321416943886, "score": 0.9826241016408392, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "wellsville missouri", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "wellsville", "type": "ClueToken", "weight": 1.0}, {"label": "missouri?", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001200", "qText": "where great britain located?", "SV": ["located"], "lemmaSV": ["locate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Great Britain", "cookedLabel": "Great Britain", "pageID": "13530298", "editDist": 0.0, "labelProbability": 0.545167, "logPopularity": 7.485491608030754, "score": 0.8424338456431942, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Kingdom of Great Britain", "cookedLabel": "Kingdom of Great Britain", "pageID": "158019", "editDist": 0.0, "labelProbability": 0.115033, "logPopularity": 7.289610521451167, "score": 0.1971669918471435, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "United Kingdom", "cookedLabel": "United Kingdom", "pageID": "31717", "editDist": 0.0, "labelProbability": 0.0500366, "logPopularity": 11.570967932364097, "score": 0.7038951322501689, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "great britain", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001220", "qText": "when is the last time the chicago bulls won a championship?", "SV": ["won"], "lemmaSV": ["win"], "LAT": [{"synset": "15147173", "text": "time", "specificity": "0.0", "type": "QuestionWordLAT"}, {"synset": "15184543", "text": "date", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "The Last Time (song)", "cookedLabel": "The Last Time", "pageID": "6205401", "editDist": 0.0, "labelProbability": 0.682171, "logPopularity": 4.634728988229636, "score": 0.830873613966111, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Last Time (Agnetha Fältskog song)", "cookedLabel": "The Last Time", "pageID": "5748973", "editDist": 0.0, "labelProbability": 0.682171, "logPopularity": 3.871201010907891, "score": 0.7565253155390046, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Last Time (film)", "cookedLabel": "The Last Time", "pageID": "19097189", "editDist": 0.0, "labelProbability": 0.682171, "logPopularity": 3.784189633918261, "score": 0.7467809835967109, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Last Time (Taylor Swift song)", "cookedLabel": "The Last Time", "pageID": "37818274", "editDist": 0.0, "labelProbability": 0.682171, "logPopularity": 3.713572066704308, "score": 0.7386854613410274, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Last Time (album)", "cookedLabel": "The Last Time", "pageID": "24162197", "editDist": 0.0, "labelProbability": 0.682171, "logPopularity": 3.871201010907891, "score": 0.7565253155390046, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Chicago Bulls", "cookedLabel": "Chicago Bulls", "pageID": "72866", "editDist": 0.0, "labelProbability": 0.934295, "logPopularity": 6.981934677156389, "score": 0.984617412627173, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Football League Championship", "cookedLabel": "Football League Championship", "pageID": "715008", "editDist": 0.0, "labelProbability": 0.472721, "logPopularity": 6.082218910376446, "score": 0.381212611810872, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Championship", "cookedLabel": "Championship", "pageID": "3884434", "editDist": 0.0, "labelProbability": 0.0986606, "logPopularity": 1.791759469228055, "score": 0.022051129155561185, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the last time the chicago bulls won a championship", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "last time the chicago bulls won a championship", "type": "CluePhrase", "weight": 0.99}, {"label": "last time", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001240", "qText": "where is mallorca?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Majorca", "cookedLabel": "Majorca", "pageID": "59310", "editDist": 0.0, "labelProbability": 0.813659, "logPopularity": 6.210600077024653, "score": 0.9430392421046333, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "RCD Mallorca", "cookedLabel": "RCD Mallorca", "pageID": "322630", "editDist": 0.0, "labelProbability": 0.143122, "logPopularity": 7.300472814267799, "score": 0.35265837803377453, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "mallorca", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001260", "qText": "what city was robert kennedy killed in?", "SV": ["killed"], "lemmaSV": ["kill"], "LAT": [{"synset": "27365", "text": "location", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8693705", "text": "urban area", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7991473", "text": "gathering", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8242502", "text": "municipality", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8643858", "text": "municipality", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "city", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Robert F. Kennedy", "cookedLabel": "Robert F. Kennedy", "pageID": "21131695", "editDist": 0.0, "labelProbability": 0.967946, "logPopularity": 5.5254529391317835, "score": 0.9689261200594848, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "robert kennedy", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr001280", "qText": "what is the closest airport to naples florida?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "3319968", "text": "facility", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "2690851", "text": "airfield", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "airport", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Naples, Florida", "cookedLabel": "Naples, Florida", "pageID": "109132", "editDist": 0.0, "labelProbability": 0.888889, "logPopularity": 6.249975242259483, "score": 0.9251916699782889, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the closest airport to naples florida", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "closest airport to naples florida", "type": "CluePhrase", "weight": 0.99}, {"label": "closest airport", "type": "CluePhrase", "weight": 0.99}, {"label": "closest", "type": "ClueToken", "weight": 1.0}, {"label": "airport", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "naples florida?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001300", "qText": "what language do they speak in argentina yahoo?", "SV": ["speak"], "lemmaSV": ["speak"], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5658174", "text": "faculty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6297048", "text": "word", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6294878", "text": "language unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5624029", "text": "ability", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6399623", "text": "text", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13831419", "text": "part", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5778661", "text": "higher cognitive process", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6376912", "text": "matter", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "language", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "They", "cookedLabel": "They", "pageID": "962806", "editDist": 0.0, "labelProbability": 0.154574, "logPopularity": 2.0794415416798357, "score": 0.02418695546554977, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "They (song)", "cookedLabel": "They", "pageID": "6129284", "editDist": 0.0, "labelProbability": 0.055205, "logPopularity": 3.871201010907891, "score": 0.016872077760925214, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "argentina yahoo", "type": "CluePhrase", "weight": 0.99}, {"label": "argentina", "type": "ClueToken", "weight": 1.0}, {"label": "yahoo", "type": "ClueToken", "weight": 1.0}] }, +{"qId": "wqr001320", "qText": "what type of planes does virgin america fly?", "SV": ["fly"], "lemmaSV": ["fly"], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7957410", "text": "biological group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "2855782", "text": "block", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847533", "text": "kind", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6830481", "text": "written symbol", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6819327", "text": "symbol", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "6831828", "text": "character", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6804229", "text": "signal", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9628463", "text": "adult", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8008892", "text": "taxonomic group", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "type", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Airplane", "cookedLabel": "Airplane", "pageID": "1396249", "editDist": 0.0, "labelProbability": 0.153099, "logPopularity": 3.828641396489095, "score": 0.06569766357637967, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Planes, Alicante", "cookedLabel": "Planes, Alicante", "pageID": "23503428", "editDist": 0.0, "labelProbability": 0.436893, "logPopularity": 3.258096538021482, "score": 0.06425363033386783, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Albion (wherry)", "cookedLabel": "Albion", "pageID": "11814367", "editDist": 0.0, "labelProbability": 0.153099, "logPopularity": 4.07753744390572, "score": 0.02956050625112393, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Plane (esotericism)", "cookedLabel": "Plane", "pageID": "1037059", "editDist": 0.0, "labelProbability": 0.153099, "logPopularity": 3.970291913552122, "score": 0.027769463513634657, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Planes (film)", "cookedLabel": "Planes", "pageID": "33619581", "editDist": 0.0, "labelProbability": 0.153099, "logPopularity": 5.056245805348308, "score": 0.051951474794694204, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Virgin America", "cookedLabel": "Virgin America", "pageID": "780894", "editDist": 0.0, "labelProbability": 0.993224, "logPopularity": 4.356708826689592, "score": 0.9455651124719849, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "planes", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr001340", "qText": "what all does google have?", "SV": [], "lemmaSV": [], "LAT": [{"text": "all", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Google", "cookedLabel": "Google", "pageID": "1092923", "editDist": 0.0, "labelProbability": 0.776912, "logPopularity": 6.598509028614515, "score": 0.9463727325949041, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Google Search", "cookedLabel": "Google Search", "pageID": "12431", "editDist": 0.0, "labelProbability": 0.126869, "logPopularity": 5.572154032177765, "score": 0.15199311696008388, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "google", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001360", "qText": "what to see outside of paris?", "SV": ["see"], "lemmaSV": ["see"], "LAT": [], "Concept": [{"fullLabel": "Outside of This", "cookedLabel": "Outside of This", "pageID": "19770019", "editDist": 3.0, "labelProbability": 0.0, "logPopularity": 4.23410650459726, "score": 0.09909197854553327, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Outside Music", "cookedLabel": "Outside Music", "pageID": "7496852", "editDist": 0.0, "labelProbability": 0.294876, "logPopularity": 4.875197323201151, "score": 0.20179120300774417, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Outside (David Bowie album)", "cookedLabel": "Outside", "pageID": "1142018", "editDist": 0.0, "labelProbability": 0.294876, "logPopularity": 4.574710978503383, "score": 0.17430408133287517, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Outside (O'Death album)", "cookedLabel": "Outside", "pageID": "35029570", "editDist": 0.0, "labelProbability": 0.294876, "logPopularity": 4.248495242049359, "score": 0.14790207526270802, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Outside (Amar album)", "cookedLabel": "Outside", "pageID": "26038888", "editDist": 0.0, "labelProbability": 0.294876, "logPopularity": 4.04305126783455, "score": 0.13303167563009013, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Outside (magazine)", "cookedLabel": "Outside", "pageID": "15602770", "editDist": 0.0, "labelProbability": 0.336771, "logPopularity": 3.9318256327243257, "score": 0.14822652123311217, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Paris", "cookedLabel": "Paris", "pageID": "22989", "editDist": 0.0, "labelProbability": 0.817026, "logPopularity": 9.647045715820404, "score": 0.9799246369048512, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "outside", "type": "ClueNE", "weight": 1.09}] }, +{"qId": "wqr001380", "qText": "what is julia gillard famous for?", "SV": [], "lemmaSV": [], "LAT": [{"text": "gillard", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Julia Gillard", "cookedLabel": "Julia Gillard", "pageID": "519437", "editDist": 0.0, "labelProbability": 0.996834, "logPopularity": 5.771441123130016, "score": 0.908025046664713, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Sham Shui Po District", "cookedLabel": "Sham Shui Po District", "pageID": "2638679", "editDist": 0.0, "labelProbability": 0.5, "logPopularity": 4.6913478822291435, "score": 0.1782132779367818, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Rampura Phul", "cookedLabel": "Rampura Phul", "pageID": "5807631", "editDist": 0.0, "labelProbability": 0.5, "logPopularity": 4.330733340286331, "score": 0.14869608916356794, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "famous for", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr001400", "qText": "who played jacob black?", "SV": ["played"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Jacob Black", "cookedLabel": "Jacob Black", "pageID": "10467799", "editDist": 0.0, "labelProbability": 0.988721, "logPopularity": 4.127134385045092, "score": 0.8455697743714826, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001420", "qText": "what currency does thailand use?", "SV": ["use"], "lemmaSV": ["use"], "LAT": [{"synset": "4923519", "text": "property", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13394134", "text": "medium of exchange", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7275291", "text": "standard", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4772610", "text": "prevalence", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5051824", "text": "temporal arrangement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5051679", "text": "temporal property", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4771667", "text": "generality", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13598374", "text": "system of measurement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5057266", "text": "presentness", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5053160", "text": "timing", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "currency", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Thailand", "cookedLabel": "Thailand", "pageID": "30128", "editDist": 0.0, "labelProbability": 0.705665, "logPopularity": 9.175541866433488, "score": 0.9835247469770341, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Thailand national football team", "cookedLabel": "Thailand national football team", "pageID": "1110063", "editDist": 0.0, "labelProbability": 0.0655873, "logPopularity": 6.97914527506881, "score": 0.23929371727969667, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "thailand", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001440", "qText": "who plays mary jane in spiderman 3?", "SV": ["plays"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Mary Jane Watson", "cookedLabel": "Mary Jane Watson", "pageID": "281687", "editDist": 0.0, "labelProbability": 0.492584, "logPopularity": 4.219507705176107, "score": 0.2973856389120331, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Cannabis (drug)", "cookedLabel": "Cannabis", "pageID": "1481886", "editDist": 0.0, "labelProbability": 0.147541, "logPopularity": 4.852030263919617, "score": 0.11241948044520658, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Mary Jane Croft", "cookedLabel": "Mary Jane Croft", "pageID": "1874734", "editDist": 0.0, "labelProbability": 0.147541, "logPopularity": 4.31748811353631, "score": 0.08417083075008917, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Untouchable", "cookedLabel": "The Untouchable", "pageID": "5041167", "editDist": 0.0, "labelProbability": 0.147541, "logPopularity": 4.31748811353631, "score": 0.08417083075008917, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "So Far, So Good... So What!", "cookedLabel": "So Far, So Good... So What!", "pageID": "60300", "editDist": 0.0, "labelProbability": 0.147541, "logPopularity": 4.812184355372417, "score": 0.11005600959212283, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Spider-Man 3", "cookedLabel": "Spider-Man 3", "pageID": "702117", "editDist": 0.0, "labelProbability": 0.928643, "logPopularity": 4.48863636973214, "score": 0.8376732727814729, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "spiderman 3", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001460", "qText": "who is the voice of lois from family guy?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9794206", "text": "advocate", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4923519", "text": "property", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7041860", "text": "tune", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10435383", "text": "performer", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9639952", "text": "entertainer", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13819354", "text": "linguistic relation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "44888", "text": "implementation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4990371", "text": "sound property", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7298313", "text": "happening", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13818991", "text": "grammatical relation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "173531", "text": "means", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5208927", "text": "physical ability", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10619214", "text": "singer", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6262268", "text": "communication", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10360025", "text": "musician", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4988388", "text": "sound", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7385893", "text": "sound", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7154581", "text": "expression", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5207437", "text": "ability", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "voice", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Whois", "cookedLabel": "Whois", "pageID": "4315433", "editDist": 0.0, "labelProbability": 0.0673077, "logPopularity": 3.1780538303479458, "score": 0.031085894047283118, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lois Griffin", "cookedLabel": "Lois Griffin", "pageID": "913759", "editDist": 0.0, "labelProbability": 0.565673, "logPopularity": 4.204692619390966, "score": 0.36988417502777904, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lois Capps", "cookedLabel": "Lois Capps", "pageID": "408884", "editDist": 0.0, "labelProbability": 0.111604, "logPopularity": 5.225746673713202, "score": 0.11844687769507006, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lois McMaster Bujold", "cookedLabel": "Lois McMaster Bujold", "pageID": "18733", "editDist": 0.0, "labelProbability": 0.111604, "logPopularity": 4.77912349311153, "score": 0.0931986907591393, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lois Hole", "cookedLabel": "Lois Hole", "pageID": "938524", "editDist": 0.0, "labelProbability": 0.111604, "logPopularity": 4.727387818712341, "score": 0.09060826457542856, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lois Lowry", "cookedLabel": "Lois Lowry", "pageID": "199942", "editDist": 0.0, "labelProbability": 0.111604, "logPopularity": 4.9344739331306915, "score": 0.10138029943961649, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Family Guy", "cookedLabel": "Family Guy", "pageID": "187586", "editDist": 0.0, "labelProbability": 0.964212, "logPopularity": 6.248042874508429, "score": 0.9458401875550986, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the voice of lois from family guy", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "voice of lois from family guy", "type": "CluePhrase", "weight": 0.99}, {"label": "voice", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "lois", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr001480", "qText": "what tourist attractions are in houston texas?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "7303344", "text": "social event", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6631572", "text": "show", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5858316", "text": "feature", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "34512", "text": "phenomenon", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "11479041", "text": "force", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9639952", "text": "entertainer", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "11428673", "text": "natural phenomenon", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "11439518", "text": "physical phenomenon", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5857567", "text": "property", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "attractions", "specificity": "0.0", "type": "LAT"}, {"text": "attraction", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Tourist attraction", "cookedLabel": "Tourist attraction", "pageID": "99863", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.5263605246161616, "score": 0.006723718697320666, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Houston", "cookedLabel": "Houston", "pageID": "13774", "editDist": 0.0, "labelProbability": 0.986784, "logPopularity": 8.83127373772255, "score": 0.9849144990612584, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "houston texas", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001500", "qText": "what does annie leibovitz do?", "SV": ["do"], "lemmaSV": ["do"], "LAT": [{"text": "leibovitz", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Annie Leibovitz", "cookedLabel": "Annie Leibovitz", "pageID": "18943868", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.564348191467836, "score": 0.8292138234969874, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001520", "qText": "what is the zip code for trenton ohio?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6371284", "text": "writing", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6365164", "text": "coding system", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6360590", "text": "written communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "code", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "ZIP code", "cookedLabel": "ZIP code", "pageID": "51550", "editDist": 0.0, "labelProbability": 0.965969, "logPopularity": 10.142898076314209, "score": 0.9915968566201331, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Genetics", "cookedLabel": "Genetics", "pageID": "12266", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 5.991464547107982, "score": 0.9266078744173097, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Ironton, Ohio", "cookedLabel": "Ironton, Ohio", "pageID": "129509", "editDist": 2.0, "labelProbability": 0.0, "logPopularity": 5.575949103146316, "score": 0.1527437025093599, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "the zip code for trenton ohio", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "zip code for trenton ohio", "type": "CluePhrase", "weight": 0.99}, {"label": "code for", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001540", "qText": "who is kobe bryant wife bio?", "SV": [], "lemmaSV": [], "LAT": [{"text": "bio", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Whois", "cookedLabel": "Whois", "pageID": "4315433", "editDist": 0.0, "labelProbability": 0.0673077, "logPopularity": 3.1780538303479458, "score": 0.031085894047283118, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Kobe Bryant", "cookedLabel": "Kobe Bryant", "pageID": "246185", "editDist": 0.0, "labelProbability": 0.977755, "logPopularity": 5.0689042022202315, "score": 0.8676040761673899, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "who is", "type": "ClueNE", "weight": 1.11}, {"label": "kobe bryant wife bio", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "wife", "type": "ClueToken", "weight": 1.0}, {"label": "bio", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr001560", "qText": "who was charles darwin married to?", "SV": ["married"], "lemmaSV": ["marry"], "LAT": [], "Concept": [{"fullLabel": "Charles Darwin", "cookedLabel": "Charles Darwin", "pageID": "8145410", "editDist": 0.0, "labelProbability": 0.986471, "logPopularity": 5.817111159963204, "score": 0.9758720993822966, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001580", "qText": "where did jane austen grow up?", "SV": ["grow"], "lemmaSV": ["grow"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Jane Austen", "cookedLabel": "Jane Austen", "pageID": "15782", "editDist": 0.0, "labelProbability": 0.982964, "logPopularity": 5.19295685089021, "score": 0.9647484200772398, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Grow Up (Svoy album)", "cookedLabel": "Grow Up", "pageID": "32182839", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.430816798843313, "score": 0.04643010512375539, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Grow Up, Tony Phillips", "cookedLabel": "Grow Up, Tony Phillips", "pageID": "41237889", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.2188758248682006, "score": 0.022990485979290737, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Grow Up (book)", "cookedLabel": "Grow Up", "pageID": "11645304", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.6635616461296463, "score": 0.02981111954944062, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Grow Up (The Queers album)", "cookedLabel": "Grow Up", "pageID": "8796937", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.9889840465642745, "score": 0.03600738939439352, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "grow up", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001600", "qText": "what time does target in alhambra close?", "SV": ["close"], "lemmaSV": ["close"], "LAT": [{"synset": "5097645", "text": "magnitude", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4923519", "text": "property", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "15137796", "text": "time period", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5824748", "text": "datum", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "15269461", "text": "moment", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4990371", "text": "sound property", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "15205381", "text": "point", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5824916", "text": "reading", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "15249282", "text": "term", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7300108", "text": "experience", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13597072", "text": "fundamental quantity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7298313", "text": "happening", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4998633", "text": "rhythmicity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7323507", "text": "case", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5824413", "text": "information", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5100843", "text": "dimension", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "time", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Target Corporation", "cookedLabel": "Target Corporation", "pageID": "18581242", "editDist": 0.0, "labelProbability": 0.688924, "logPopularity": 5.0369526024136295, "score": 0.821892941151469, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Target Books", "cookedLabel": "Target Books", "pageID": "930311", "editDist": 0.0, "labelProbability": 0.11761, "logPopularity": 5.241747015059643, "score": 0.27409486180995263, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Target (UK TV series)", "cookedLabel": "Target", "pageID": "8735505", "editDist": 0.0, "labelProbability": 0.11761, "logPopularity": 5.043425116919247, "score": 0.25106580573413756, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Target Field", "cookedLabel": "Target Field", "pageID": "4932680", "editDist": 0.0, "labelProbability": 0.11761, "logPopularity": 4.663439094112067, "score": 0.21066469210674765, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Target Center", "cookedLabel": "Target Center", "pageID": "237003", "editDist": 0.0, "labelProbability": 0.11761, "logPopularity": 5.147494476813453, "score": 0.26298803860427983, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Alhambra", "cookedLabel": "Alhambra", "pageID": "30543", "editDist": 0.0, "labelProbability": 0.753071, "logPopularity": 4.624972813284271, "score": 0.6412571396044241, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Alhambra, California", "cookedLabel": "Alhambra, California", "pageID": "107595", "editDist": 0.0, "labelProbability": 0.102766, "logPopularity": 5.54907608489522, "score": 0.05521313576151007, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "target in alhambra", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "target", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001620", "qText": "what are the best places to go in germany?", "SV": ["go"], "lemmaSV": ["go"], "LAT": [{"synset": "8637195", "text": "public square", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6360590", "text": "written communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8508037", "text": "address", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "720746", "text": "duty", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6494090", "text": "item", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "576778", "text": "work", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13831419", "text": "part", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6374360", "text": "writing", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8658688", "text": "vicinity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5130681", "text": "extent", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "24900", "text": "state", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5135784", "text": "area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8514304", "text": "area", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8576500", "text": "residence", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6411914", "text": "passage", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "583425", "text": "occupation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8665897", "text": "section", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8637636", "text": "point", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13968971", "text": "status", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13943868", "text": "condition", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13948785", "text": "situation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13970595", "text": "social station", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5097645", "text": "magnitude", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "721817", "text": "function", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8596234", "text": "geographic point", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8670545", "text": "space", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6403644", "text": "section", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4923519", "text": "property", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8691133", "text": "tract", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"text": "places", "specificity": "0.0", "type": "LAT"}, {"text": "place", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Tony Award for Best Play", "cookedLabel": "Tony Award for Best Play", "pageID": "250941", "editDist": 3.0, "labelProbability": 0.0, "logPopularity": 2.3978952727983707, "score": 0.023125298297074125, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Best Play ESPY Award", "cookedLabel": "Best Play ESPY Award", "pageID": "10975973", "editDist": 3.0, "labelProbability": 0.0, "logPopularity": 2.1972245773362196, "score": 0.02055601553346779, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Helpmann Award for Best Play", "cookedLabel": "Helpmann Award for Best Play", "pageID": "33286338", "editDist": 3.0, "labelProbability": 0.0, "logPopularity": 2.1972245773362196, "score": 0.02055601553346779, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "The Best (song)", "cookedLabel": "The Best", "pageID": "5519593", "editDist": 0.0, "labelProbability": 0.292556, "logPopularity": 4.804021044733257, "score": 0.19333164156720659, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Best (Despina Vandi album)", "cookedLabel": "The Best", "pageID": "15916142", "editDist": 0.0, "labelProbability": 0.158684, "logPopularity": 4.976733742420574, "score": 0.1256234154068918, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Best (Portuguese footballer)", "cookedLabel": "Best", "pageID": "18953026", "editDist": 0.0, "labelProbability": 0.158684, "logPopularity": 4.532599493153256, "score": 0.0991506029418958, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Best, Netherlands", "cookedLabel": "Best, Netherlands", "pageID": "118705", "editDist": 0.0, "labelProbability": 0.158684, "logPopularity": 4.624972813284271, "score": 0.10421212240951007, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Best (Mika Nakashima album)", "cookedLabel": "Best", "pageID": "6123928", "editDist": 0.0, "labelProbability": 0.158684, "logPopularity": 4.736198448394496, "score": 0.11060845654008894, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Take-out", "cookedLabel": "Take-out", "pageID": "326234", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.258096538021482, "score": 0.023525035528552937, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Germany", "cookedLabel": "Germany", "pageID": "11867", "editDist": 0.0, "labelProbability": 0.731908, "logPopularity": 11.210644004861829, "score": 0.9882824313361198, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the best places to go in germany", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "best places to go in germany", "type": "CluePhrase", "weight": 0.99}, {"label": "the best", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr001640", "qText": "where did pizarro land?", "SV": ["land"], "lemmaSV": ["land"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Francisco Pizarro", "cookedLabel": "Francisco Pizarro", "pageID": "55271", "editDist": 0.0, "labelProbability": 0.627421, "logPopularity": 4.8283137373023015, "score": 0.7542382553020757, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "David Pizarro", "cookedLabel": "David Pizarro", "pageID": "2372903", "editDist": 0.0, "labelProbability": 0.0810398, "logPopularity": 4.948759890378168, "score": 0.09081393176116873, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Claudio Pizarro", "cookedLabel": "Claudio Pizarro", "pageID": "1149234", "editDist": 0.0, "labelProbability": 0.116972, "logPopularity": 5.1298987149230735, "score": 0.1160998607878938, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "pizarro", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001660", "qText": "what is there to do in mt baldy california?", "SV": ["do"], "lemmaSV": ["do"], "LAT": [], "Concept": [{"fullLabel": "Time management", "cookedLabel": "Time management", "pageID": "31092", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.5553480614894135, "score": 0.7453702846431538, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Mount Baldy (Arizona)", "cookedLabel": "Mount Baldy", "pageID": "11869346", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.897839799950911, "score": 0.060536868671169725, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Mount Baldy, California", "cookedLabel": "Mount Baldy, California", "pageID": "24855182", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.795790545596741, "score": 0.0571469495266546, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Mount Baldy Ski Area", "cookedLabel": "Mount Baldy Ski Area", "pageID": "10878447", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.0943445622221, "score": 0.03826717146117365, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Mount Baldy (Nevada)", "cookedLabel": "Mount Baldy", "pageID": "29651951", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.828641396489095, "score": 0.03281312068885659, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Mount Baldy (Alberta)", "cookedLabel": "Mount Baldy", "pageID": "1474567", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.9318256327243257, "score": 0.03483575892136317, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "to do", "type": "ClueNE", "weight": 1.11}, {"label": "mt baldy california", "type": "CluePhrase", "weight": 0.99}, {"label": "california", "type": "ClueToken", "weight": 1.0}] }, +{"qId": "wqr001680", "qText": "who is the ravens quarterback 2012?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "15184543", "text": "date", "specificity": "-2.0", "type": "LAT"}], "Concept": [{"fullLabel": "Whois", "cookedLabel": "Whois", "pageID": "4315433", "editDist": 0.0, "labelProbability": 0.0673077, "logPopularity": 3.1780538303479458, "score": 0.031085894047283118, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Ravens", "cookedLabel": "The Ravens", "pageID": "1081322", "editDist": 0.0, "labelProbability": 0.81746, "logPopularity": 4.174387269895637, "score": 0.6471288237888566, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "2012", "cookedLabel": "2012", "pageID": "47374", "editDist": 0.0, "labelProbability": 0.370218, "logPopularity": 2.833213344056216, "score": 0.22133110536291262, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "2012 phenomenon", "cookedLabel": "2012 phenomenon", "pageID": "21538638", "editDist": 0.0, "labelProbability": 0.062167, "logPopularity": 4.219507705176107, "score": 0.055828191938404, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "2012 (film)", "cookedLabel": "2012", "pageID": "18436536", "editDist": 0.0, "labelProbability": 0.128225, "logPopularity": 4.736198448394496, "score": 0.09846614463200648, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "the ravens quarterback 2012", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "ravens quarterback 2012", "type": "CluePhrase", "weight": 0.99}, {"label": "quarterback", "type": "ClueToken", "weight": 1.0}, {"label": "2012", "type": "ClueNE", "weight": 2.3000000000000003}] }, +{"qId": "wqr001700", "qText": "who plays stella in coronation street?", "SV": ["plays"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Estelle (musician)", "cookedLabel": "Estelle", "pageID": "3854523", "editDist": 0.0, "labelProbability": 0.910677, "logPopularity": 5.459585514144159, "score": 0.6942599195528122, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "List of Seinfeld minor characters", "cookedLabel": "stella", "pageID": "571523", "editDist": 0.0, "labelProbability": 0.910677, "logPopularity": 4.477336814478207, "score": 0.5574384809980283, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Princess Estelle, Duchess of Östergötland", "cookedLabel": "Princess Estelle, Duchess of Östergötland", "pageID": "34853027", "editDist": 0.0, "labelProbability": 0.910677, "logPopularity": 4.406719247264253, "score": 0.5469618964089914, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Estelle, Louisiana", "cookedLabel": "Estelle, Louisiana", "pageID": "115613", "editDist": 0.0, "labelProbability": 0.910677, "logPopularity": 4.30406509320417, "score": 0.5316604686865143, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Don Estelle", "cookedLabel": "Don Estelle", "pageID": "287068", "editDist": 0.0, "labelProbability": 0.910677, "logPopularity": 4.418840607796598, "score": 0.5487634046109606, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Coronation Street", "cookedLabel": "Coronation Street", "pageID": "6851", "editDist": 0.0, "labelProbability": 0.995472, "logPopularity": 6.066108090103747, "score": 0.9475819375105158, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "stella", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr001720", "qText": "what did cam newton do?", "SV": ["do"], "lemmaSV": ["do"], "LAT": [{"synset": "10320928", "text": "mathematician", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13604927", "text": "unit of measurement", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "13597304", "text": "definite quantity", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10447768", "text": "physicist", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10580065", "text": "scientist", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13624308", "text": "force unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "newton", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Cam Newton", "cookedLabel": "Cam Newton", "pageID": "9521131", "editDist": 0.0, "labelProbability": 0.650866, "logPopularity": 5.0689042022202315, "score": 0.5690556655271514, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Cam Newton (ice hockey)", "cookedLabel": "Cam Newton", "pageID": "22349459", "editDist": 0.0, "labelProbability": 0.262997, "logPopularity": 4.30406509320417, "score": 0.04975335974685033, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001740", "qText": "what did neil say on the moon?", "SV": ["say"], "lemmaSV": ["say"], "LAT": [{"text": "neil", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Neil Armstrong", "cookedLabel": "Neil Armstrong", "pageID": "21247", "editDist": 0.0, "labelProbability": 0.110849, "logPopularity": 5.111987788356544, "score": 0.029334343126575808, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Neil Bush", "cookedLabel": "Neil Bush", "pageID": "243759", "editDist": 0.0, "labelProbability": 0.0687893, "logPopularity": 4.330733340286331, "score": 0.015348128311558275, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Neil", "cookedLabel": "Neil", "pageID": "1338760", "editDist": 0.0, "labelProbability": 0.112421, "logPopularity": 1.6094379124341003, "score": 0.009877386550343368, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "On the Moon", "cookedLabel": "On the Moon", "pageID": "7219118", "editDist": 0.0, "labelProbability": 0.534162, "logPopularity": 3.784189633918261, "score": 0.3555170376195336, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "neil", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001760", "qText": "what were marco polo's goals?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "38116", "text": "action", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8586507", "text": "extremity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3419072", "text": "game equipment", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "64472", "text": "success", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3580409", "text": "instrumentality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "35910", "text": "accomplishment", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "63626", "text": "attainment", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "3298959", "text": "equipment", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8583557", "text": "end", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8647614", "text": "region", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "187483", "text": "score", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "goals", "specificity": "0.0", "type": "LAT"}, {"text": "goal", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Marco Polo", "cookedLabel": "Marco Polo", "pageID": "19334", "editDist": 0.0, "labelProbability": 0.880007, "logPopularity": 4.624972813284271, "score": 0.8174643786380883, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "marco polo's goals", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "goals", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr001780", "qText": "what language brazil use?", "SV": ["brazil"], "lemmaSV": ["brazil"], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5658174", "text": "faculty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6297048", "text": "word", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6294878", "text": "language unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5624029", "text": "ability", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6399623", "text": "text", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13831419", "text": "part", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5778661", "text": "higher cognitive process", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6376912", "text": "matter", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "language", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Glossary of vexillology", "cookedLabel": "Glossary of vexillology", "pageID": "113612", "editDist": 0.0, "labelProbability": 0.203481, "logPopularity": 3.4011973816621555, "score": 0.02495217404278388, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Use case", "cookedLabel": "Use case", "pageID": "300006", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.2188758248682006, "score": 0.022990485979290737, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Bisphenol A", "cookedLabel": "Bisphenol A", "pageID": "1001430", "editDist": 0.0, "labelProbability": 0.0586792, "logPopularity": 3.8501476017100584, "score": 0.016927525045779115, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "User story", "cookedLabel": "User story", "pageID": "2656549", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.70805020110221, "score": 0.01702491912837177, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Use–mention distinction", "cookedLabel": "Use–mention distinction", "pageID": "172990", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.258096538021482, "score": 0.023525035528552937, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "use", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr001800", "qText": "when will muharram start 2011?", "SV": ["start"], "lemmaSV": ["start"], "LAT": [{"synset": "15147173", "text": "time", "specificity": "0.0", "type": "QuestionWordLAT"}, {"synset": "15184543", "text": "date", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Muharram", "cookedLabel": "Muharram", "pageID": "444123", "editDist": 0.0, "labelProbability": 0.903818, "logPopularity": 2.5649493574615367, "score": 0.7376540387559763, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Day of Ashura", "cookedLabel": "Day of Ashura", "pageID": "488563", "editDist": 0.0, "labelProbability": 0.065387, "logPopularity": 4.204692619390966, "score": 0.056140632929766454, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "2011", "cookedLabel": "2011", "pageID": "36225", "editDist": 0.0, "labelProbability": 0.353935, "logPopularity": 2.6390573296152584, "score": 0.07978062557656532, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "muharram", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001820", "qText": "what currency do mexico use?", "SV": ["use"], "lemmaSV": ["use"], "LAT": [{"synset": "4923519", "text": "property", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13394134", "text": "medium of exchange", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7275291", "text": "standard", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4772610", "text": "prevalence", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5051824", "text": "temporal arrangement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5051679", "text": "temporal property", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4771667", "text": "generality", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13598374", "text": "system of measurement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5057266", "text": "presentness", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5053160", "text": "timing", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "currency", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Mexico", "cookedLabel": "Mexico", "pageID": "3966054", "editDist": 0.0, "labelProbability": 0.705114, "logPopularity": 10.004282662571022, "score": 0.9898891719082676, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001840", "qText": "what books did lincoln write?", "SV": ["write"], "lemmaSV": ["write"], "LAT": [{"synset": "4606723", "text": "work", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6360590", "text": "written communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7968050", "text": "collection", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7020800", "text": "dramatic composition", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6646883", "text": "information", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13424816", "text": "record", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6648784", "text": "fact", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13424504", "text": "document", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6611268", "text": "message", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6374360", "text": "writing", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4014270", "text": "product", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6601855", "text": "publication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6441260", "text": "sacred text", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6403644", "text": "section", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3133774", "text": "creation", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"text": "books", "specificity": "0.0", "type": "LAT"}, {"text": "book", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Lincolnshire", "cookedLabel": "Lincolnshire", "pageID": "53295", "editDist": 0.0, "labelProbability": 0.117749, "logPopularity": 8.146709052203319, "score": 0.6834428177880589, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lincoln, Nebraska", "cookedLabel": "Lincoln, Nebraska", "pageID": "17653", "editDist": 0.0, "labelProbability": 0.117749, "logPopularity": 7.400009517162692, "score": 0.5797208566933856, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Abraham Lincoln", "cookedLabel": "Abraham Lincoln", "pageID": "307", "editDist": 0.0, "labelProbability": 0.117749, "logPopularity": 6.089044875446846, "score": 0.38581533885682096, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lincoln, England", "cookedLabel": "Lincoln, England", "pageID": "17880", "editDist": 0.0, "labelProbability": 0.117749, "logPopularity": 6.639875833826536, "score": 0.46643947108922923, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Lincoln City F.C.", "cookedLabel": "Lincoln City F.C.", "pageID": "451163", "editDist": 0.0, "labelProbability": 0.117749, "logPopularity": 8.052296499538647, "score": 0.6710620781440747, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "lincoln", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001860", "qText": "where does kirk cameron live now?", "SV": ["live"], "lemmaSV": ["live"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Kirk Cameron", "cookedLabel": "Kirk Cameron", "pageID": "20398264", "editDist": 0.0, "labelProbability": 0.979936, "logPopularity": 4.812184355372417, "score": 0.95550994552952, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Rolling Stones, Now!", "cookedLabel": "The Rolling Stones, Now!", "pageID": "1365981", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.543294782270004, "score": 0.13527332344325466, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now TV", "cookedLabel": "Now TV", "pageID": "2857006", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.430816798843313, "score": 0.1275717260871764, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now! (Bobby Hutcherson album)", "cookedLabel": "Now!", "pageID": "2580911", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.248495242049359, "score": 0.1158846124007123, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (Fireflight album)", "cookedLabel": "Now", "pageID": "34423465", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.343805421853684, "score": 0.1218734455168399, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (newspaper)", "cookedLabel": "Now", "pageID": "1058750", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.532599493153256, "score": 0.1345244471522166, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "now", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr001880", "qText": "what disease did abe lincoln have?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "14085287", "text": "illness", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14057659", "text": "physical condition", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "14075399", "text": "pathological state", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24900", "text": "state", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "14075528", "text": "ill health", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13943868", "text": "condition", "specificity": "-5.0", "type": "WordnetLAT"}, {"text": "disease", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Abraham Lincoln", "cookedLabel": "Abraham Lincoln", "pageID": "307", "editDist": 0.0, "labelProbability": 0.869471, "logPopularity": 6.089044875446846, "score": 0.9652876094180043, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "abe lincoln", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr001900", "qText": "what going on in afghanistan right now?", "SV": ["going"], "lemmaSV": ["go"], "LAT": [], "Concept": [{"fullLabel": "Going On", "cookedLabel": "Going On", "pageID": "18511618", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.9512437185814275, "score": 0.03522960933689275, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Afghanistan", "cookedLabel": "Afghanistan", "pageID": "737", "editDist": 0.0, "labelProbability": 0.746978, "logPopularity": 8.384347278082808, "score": 0.9431281412932907, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Right Now (Atomic Kitten album)", "cookedLabel": "Right Now", "pageID": "3409298", "editDist": 0.0, "labelProbability": 0.14878, "logPopularity": 5.10594547390058, "score": 0.12918088923564972, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Right Now (Leon Jackson album)", "cookedLabel": "Right Now", "pageID": "19354950", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.812184355372417, "score": 0.057679244816714784, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Right Now (Rihanna song)", "cookedLabel": "Right Now", "pageID": "37535311", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.543294782270004, "score": 0.04951112306188091, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Right Now (Herbie Mann song)", "cookedLabel": "Right Now", "pageID": "9209791", "editDist": 0.0, "labelProbability": 0.160366, "logPopularity": 4.174387269895637, "score": 0.08211961058034786, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Right Now (Korn song)", "cookedLabel": "Right Now", "pageID": "6012234", "editDist": 0.0, "labelProbability": 0.290854, "logPopularity": 3.9889840465642745, "score": 0.1272670428128275, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001920", "qText": "where is chowchilla located?", "SV": ["located"], "lemmaSV": ["locate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Chowchilla, California", "cookedLabel": "Chowchilla, California", "pageID": "107709", "editDist": 0.0, "labelProbability": 0.576923, "logPopularity": 4.77912349311153, "score": 0.46849432041420475, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Chowchilla River", "cookedLabel": "Chowchilla River", "pageID": "5017862", "editDist": 0.0, "labelProbability": 0.0903399, "logPopularity": 4.61512051684126, "score": 0.07862560117978776, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Chowchilla Airport", "cookedLabel": "Chowchilla Airport", "pageID": "7323766", "editDist": 0.0, "labelProbability": 0.0903399, "logPopularity": 4.1588830833596715, "score": 0.06094475304922138, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Chowchilla", "cookedLabel": "Chowchilla", "pageID": "12690456", "editDist": 0.0, "labelProbability": 0.330054, "logPopularity": 3.7376696182833684, "score": 0.28907595287084764, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "chowchilla", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr001940", "qText": "where does brian williams live?", "SV": ["live"], "lemmaSV": ["live"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Brian Williams", "cookedLabel": "Brian Williams", "pageID": "8269855", "editDist": 0.0, "labelProbability": 0.742176, "logPopularity": 4.882801922586371, "score": 0.88251710543483, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Brian Williams (sportscaster)", "cookedLabel": "Brian Williams", "pageID": "1231896", "editDist": 0.0, "labelProbability": 0.0674049, "logPopularity": 4.406719247264253, "score": 0.08653846245236564, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr001960", "qText": "what was the cause of death for sage stallone?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "1183965", "text": "due process", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "576778", "text": "work", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6735202", "text": "statement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "1930", "text": "physical entity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "797381", "text": "undertaking", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6611268", "text": "message", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7338522", "text": "origin", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "1082290", "text": "group action", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7298313", "text": "happening", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "1187304", "text": "proceeding", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6751030", "text": "explanation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7305628", "text": "beginning", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6752932", "text": "justification", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "799539", "text": "venture", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "cause", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "The Cause", "cookedLabel": "The Cause", "pageID": "1555536", "editDist": 0.0, "labelProbability": 0.759615, "logPopularity": 2.302585092994046, "score": 0.31378069925397956, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Cause of death", "cookedLabel": "Cause of death", "pageID": "791114", "editDist": 0.0, "labelProbability": 0.363077, "logPopularity": 1.791759469228055, "score": 0.03402476362264536, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Sage Stallone", "cookedLabel": "Sage Stallone", "pageID": "3253292", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.6443908991413725, "score": 0.8490903710890593, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the cause of death for sage stallone", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "cause of death for sage stallone", "type": "CluePhrase", "weight": 0.99}, {"label": "death for sage stallone", "type": "CluePhrase", "weight": 0.99}] }, +{"qId": "wqr001980", "qText": "what has been discovered on mars so far?", "SV": ["discovered"], "lemmaSV": ["discover"], "LAT": [], "Concept": [{"fullLabel": "Has Been", "cookedLabel": "Has Been", "pageID": "1105249", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.828641396489095, "score": 0.77522331235702, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Mars", "cookedLabel": "Mars", "pageID": "14640471", "editDist": 0.0, "labelProbability": 0.632056, "logPopularity": 5.84354441703136, "score": 0.680421338259525, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Mars (mythology)", "cookedLabel": "Mars", "pageID": "19638032", "editDist": 0.0, "labelProbability": 0.12615, "logPopularity": 3.9512437185814275, "score": 0.024340886274224788, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "So Far (album)", "cookedLabel": "So Far", "pageID": "4320167", "editDist": 0.0, "labelProbability": 0.494993, "logPopularity": 4.574710978503383, "score": 0.34624469132521296, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "So Far... The Best of Sinéad O'Connor", "cookedLabel": "So Far... The Best of Sinéad O'Connor", "pageID": "5870699", "editDist": 0.0, "labelProbability": 0.101574, "logPopularity": 3.9318256327243257, "score": 0.05574218696743388, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Grateful Dead: So Far", "cookedLabel": "Grateful Dead: So Far", "pageID": "11580269", "editDist": 0.0, "labelProbability": 0.260372, "logPopularity": 3.4011973816621555, "score": 0.08180149482942516, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Faust So Far", "cookedLabel": "Faust So Far", "pageID": "3308190", "editDist": 0.0, "labelProbability": 0.101574, "logPopularity": 3.9512437185814275, "score": 0.05635860176312338, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "So Far (Faust song)", "cookedLabel": "So Far", "pageID": "26348839", "editDist": 0.0, "labelProbability": 0.101574, "logPopularity": 3.9512437185814275, "score": 0.05635860176312338, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "so far", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002000", "qText": "who is president of france?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "587299", "text": "position", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10182584", "text": "head", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10488931", "text": "presiding officer", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10184340", "text": "head of state", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9633690", "text": "communicator", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10541628", "text": "representative", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "597922", "text": "presidency", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "10371605", "text": "negotiator", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9646208", "text": "leader", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9778216", "text": "academic administrator", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "583425", "text": "occupation", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9790372", "text": "administrator", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9985785", "text": "corporate executive", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10089452", "text": "executive", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "president", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Whois", "cookedLabel": "Whois", "pageID": "4315433", "editDist": 0.0, "labelProbability": 0.0673077, "logPopularity": 3.1780538303479458, "score": 0.031085894047283118, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "President of France", "cookedLabel": "President of France", "pageID": "24899", "editDist": 0.0, "labelProbability": 0.97594, "logPopularity": 4.48863636973214, "score": 0.8059721771697235, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "president of france?", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr002020", "qText": "what was eli whitney education?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "8131836", "text": "department", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-10.0", "type": "WordnetLAT"}, {"synset": "8237635", "text": "division", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8094128", "text": "administrative unit", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "4928188", "text": "inheritance", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8136796", "text": "government department", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8140150", "text": "executive department", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "611221", "text": "profession", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5760541", "text": "learning", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-8.0", "type": "WordnetLAT"}, {"synset": "583425", "text": "occupation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "4928931", "text": "upbringing", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5709891", "text": "basic cognitive process", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-9.0", "type": "WordnetLAT"}, {"synset": "8139116", "text": "federal department", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "education", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Eli Whitney", "cookedLabel": "Eli Whitney", "pageID": "9732", "editDist": 0.0, "labelProbability": 0.961735, "logPopularity": 4.442651256490317, "score": 0.8069882144943348, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "eli whitney education", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "education", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr002040", "qText": "what american penny is worth money?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "13409418", "text": "coin", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13604927", "text": "unit of measurement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13407086", "text": "currency", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13597304", "text": "definite quantity", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "13394134", "text": "medium of exchange", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "13684808", "text": "fractional monetary unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13598374", "text": "system of measurement", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "13409050", "text": "coinage", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13625961", "text": "monetary unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7275291", "text": "standard", "specificity": "-5.0", "type": "WordnetLAT"}, {"text": "penny", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Penny (United States coin)", "cookedLabel": "Penny", "pageID": "164092", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.276666119016055, "score": 0.6014212784537785, "getByLAT": 1, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "american penny", "type": "ClueNE", "weight": 1.6}, {"label": "worth money", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "worth", "type": "ClueToken", "weight": 1.0}, {"label": "money", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr002060", "qText": "what kind of government does spain have now?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "kind", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Government", "cookedLabel": "Government", "pageID": "12229", "editDist": 0.0, "labelProbability": 0.139222, "logPopularity": 5.5093883366279774, "score": 0.15315169535105763, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Spain", "cookedLabel": "Spain", "pageID": "26667", "editDist": 0.0, "labelProbability": 0.708413, "logPopularity": 10.559711378991475, "score": 0.9928424271025632, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Rolling Stones, Now!", "cookedLabel": "The Rolling Stones, Now!", "pageID": "1365981", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.543294782270004, "score": 0.13527332344325466, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now! (Bobby Hutcherson album)", "cookedLabel": "Now!", "pageID": "2580911", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.248495242049359, "score": 0.1158846124007123, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now TV", "cookedLabel": "Now TV", "pageID": "2857006", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.430816798843313, "score": 0.1275717260871764, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (Fireflight album)", "cookedLabel": "Now", "pageID": "34423465", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.343805421853684, "score": 0.1218734455168399, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (newspaper)", "cookedLabel": "Now", "pageID": "1058750", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.532599493153256, "score": 0.1345244471522166, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "now", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002080", "qText": "what art movement was vincent van gogh apart of?", "SV": ["gogh"], "lemmaSV": ["Gogh"], "LAT": [{"synset": "13494300", "text": "elimination", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "576778", "text": "work", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "11439518", "text": "physical phenomenon", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13461236", "text": "bodily process", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "13487789", "text": "discharge", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "6202938", "text": "attitude", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "11428673", "text": "natural phenomenon", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "34512", "text": "phenomenon", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "3580409", "text": "instrumentality", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "797381", "text": "undertaking", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "11511038", "text": "optical phenomenon", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7298313", "text": "happening", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "2680572", "text": "action", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3743963", "text": "mechanism", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "29976", "text": "process", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "799539", "text": "venture", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13480291", "text": "defecation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3187746", "text": "device", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13547313", "text": "organic process", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "191991", "text": "change", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6206319", "text": "inclination", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7051211", "text": "musical composition", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "11510863", "text": "optical illusion", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "38116", "text": "action", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "movement", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Art movement", "cookedLabel": "Art movement", "pageID": "228568", "editDist": 0.0, "labelProbability": 0.790698, "logPopularity": 3.091042453358316, "score": 0.16840087970345582, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Vincent van Gogh", "cookedLabel": "Vincent van Gogh", "pageID": "32603", "editDist": 0.0, "labelProbability": 0.993177, "logPopularity": 6.154858094016418, "score": 0.9807994905546668, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Apartness relation", "cookedLabel": "Apartness relation", "pageID": "7844336", "editDist": 0.0, "labelProbability": 0.818182, "logPopularity": 1.6094379124341003, "score": 0.2830910245062143, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "2008–09 Chelsea F.C. season", "cookedLabel": "2008–09 Chelsea F.C. season", "pageID": "17575455", "editDist": 0.0, "labelProbability": 0.121212, "logPopularity": 6.837332814685591, "score": 0.1211024385658796, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "apart", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002100", "qText": "where are brembo brakes from?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [], "Clue": [{"label": "brembo brakes", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "brembo", "type": "ClueToken", "weight": 1.0}, {"label": "brakes", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr002120", "qText": "what organism did mendel use?", "SV": ["use"], "lemmaSV": ["use"], "LAT": [{"synset": "8452398", "text": "system", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4258", "text": "living thing", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "organism", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Gregor Mendel", "cookedLabel": "Gregor Mendel", "pageID": "12562", "editDist": 0.0, "labelProbability": 0.885144, "logPopularity": 4.68213122712422, "score": 0.9018774668477498, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "L. Mendel Rivers", "cookedLabel": "L. Mendel Rivers", "pageID": "962771", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.543294782270004, "score": 0.12360671139896476, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Mendel Jackson Davis", "cookedLabel": "Mendel Jackson Davis", "pageID": "11232293", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.48863636973214, "score": 0.12009777896255329, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Mitchell Schwartz", "cookedLabel": "Mitchell Schwartz", "pageID": "34940839", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.532599493153256, "score": 0.1229132413270456, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Felix Mendelssohn", "cookedLabel": "Felix Mendelssohn", "pageID": "76370", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.736198448394496, "score": 0.13670041096528363, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "mendel", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr002140", "qText": "what did brittany murphy died of?", "SV": ["died"], "lemmaSV": ["die"], "LAT": [{"synset": "21445", "text": "food", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7581905", "text": "foodstuff", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7723196", "text": "vegetable", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7726028", "text": "root vegetable", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7725752", "text": "solanaceous vegetable", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7582428", "text": "starches", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7571428", "text": "food", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7721456", "text": "produce", "specificity": "-3.0", "type": "WordnetLAT"}, {"text": "murphy", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Brittany Murphy", "cookedLabel": "Brittany Murphy", "pageID": "166777", "editDist": 0.0, "labelProbability": 0.998574, "logPopularity": 5.231108616854587, "score": 0.8779929805163729, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002160", "qText": "what was robert hooke's contributions to science?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "1088005", "text": "giving", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "1108713", "text": "transaction", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "13273872", "text": "transferred property", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13274154", "text": "acquisition", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13352213", "text": "sum", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "1103863", "text": "publication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "787849", "text": "attempt", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "1082290", "text": "group action", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13285910", "text": "gift", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "32912", "text": "possession", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13350663", "text": "assets", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "1096649", "text": "commercial enterprise", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "1087717", "text": "sharing", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "1092370", "text": "commerce", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "1085001", "text": "distribution", "specificity": "-3.0", "type": "WordnetLAT"}, {"text": "contributions", "specificity": "0.0", "type": "LAT"}, {"text": "contribution", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Robert Hooke", "cookedLabel": "Robert Hooke", "pageID": "49720", "editDist": 0.0, "labelProbability": 0.997944, "logPopularity": 4.624972813284271, "score": 0.8850716192602223, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Poems by Edgar Allan Poe", "cookedLabel": "Poems by Edgar Allan Poe", "pageID": "9449219", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 2.9444389791664403, "score": 0.669855592572084, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "robert hooke's contributions to science", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "robert hooke's contributions", "type": "CluePhrase", "weight": 0.99}, {"label": "contributions", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "to science", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002180", "qText": "what was sir arthur conan doyle famous for?", "SV": [], "lemmaSV": [], "LAT": [{"text": "doyle", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Arthur Conan Doyle", "cookedLabel": "Arthur Conan Doyle", "pageID": "18951335", "editDist": 0.0, "labelProbability": 0.992414, "logPopularity": 5.986452005284438, "score": 0.9167085205418054, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Sham Shui Po District", "cookedLabel": "Sham Shui Po District", "pageID": "2638679", "editDist": 0.0, "labelProbability": 0.5, "logPopularity": 4.6913478822291435, "score": 0.1782132779367818, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Rampura Phul", "cookedLabel": "Rampura Phul", "pageID": "5807631", "editDist": 0.0, "labelProbability": 0.5, "logPopularity": 4.330733340286331, "score": 0.14869608916356794, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "sir arthur conan doyle", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr002200", "qText": "what school did burne hogarth establish?", "SV": ["establish"], "lemmaSV": ["establish"], "LAT": [{"synset": "5761561", "text": "education", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5709891", "text": "basic cognitive process", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "2916498", "text": "building", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7981699", "text": "body", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5760541", "text": "learning", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4348764", "text": "structure", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13597072", "text": "fundamental quantity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8293263", "text": "educational institution", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8070328", "text": "institution", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "15137796", "text": "time period", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8010371", "text": "animal group", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7957410", "text": "biological group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "school", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Burne Hogarth", "cookedLabel": "Burne Hogarth", "pageID": "795964", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.143134726391533, "score": 0.9403513048805165, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002220", "qText": "what years did yankees win championships?", "SV": ["win"], "lemmaSV": ["win"], "LAT": [{"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "13597072", "text": "fundamental quantity", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7991473", "text": "gathering", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "15165852", "text": "life", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "15137796", "text": "time period", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "15169331", "text": "time of life", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "years", "specificity": "0.0", "type": "LAT"}, {"text": "year", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "New York Yankees", "cookedLabel": "New York Yankees", "pageID": "4848143", "editDist": 0.0, "labelProbability": 0.643442, "logPopularity": 8.137103389639302, "score": 0.9600832390585778, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Yankee", "cookedLabel": "Yankee", "pageID": "38936", "editDist": 0.0, "labelProbability": 0.142782, "logPopularity": 2.8903717578961645, "score": 0.03714944875114816, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "yankees", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr002240", "qText": "who is the current president of dominican republic 2011?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "587299", "text": "position", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10182584", "text": "head", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10488931", "text": "presiding officer", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10184340", "text": "head of state", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9633690", "text": "communicator", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10541628", "text": "representative", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "597922", "text": "presidency", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "10371605", "text": "negotiator", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9646208", "text": "leader", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9778216", "text": "academic administrator", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "583425", "text": "occupation", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9790372", "text": "administrator", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9985785", "text": "corporate executive", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10089452", "text": "executive", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "president", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Whois", "cookedLabel": "Whois", "pageID": "4315433", "editDist": 0.0, "labelProbability": 0.0673077, "logPopularity": 3.1780538303479458, "score": 0.031085894047283118, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "United Nations Economic and Social Council", "cookedLabel": "United Nations Economic and Social Council", "pageID": "31958", "editDist": 0.0, "labelProbability": 0.47619, "logPopularity": 3.6888794541139363, "score": 0.129607165691624, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Heather Knight (educator)", "cookedLabel": "Heather Knight", "pageID": "31577192", "editDist": 0.0, "labelProbability": 0.285714, "logPopularity": 4.634728988229636, "score": 0.09863712707634434, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Mahmoud Abbas", "cookedLabel": "Mahmoud Abbas", "pageID": "232595", "editDist": 0.0, "labelProbability": 0.142857, "logPopularity": 5.10594547390058, "score": 0.07002029201560565, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "List of Presidents of the Dominican Republic", "cookedLabel": "president of dominican republic", "pageID": "451896", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.02535169073515, "score": 0.7784529784807407, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "2011", "cookedLabel": "2011", "pageID": "36225", "editDist": 0.0, "labelProbability": 0.353935, "logPopularity": 2.6390573296152584, "score": 0.1080943974551401, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the current president of dominican republic 2011", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "current president of dominican republic 2011", "type": "CluePhrase", "weight": 0.99}, {"label": "current president", "type": "ClueNE", "weight": 1.11}, {"label": "dominican republic 2011", "type": "CluePhrase", "weight": 0.99}] }, +{"qId": "wqr002260", "qText": "where did william howard taft go to high school?", "SV": ["go"], "lemmaSV": ["go"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "William Howard Taft", "cookedLabel": "William Howard Taft", "pageID": "33522", "editDist": 0.0, "labelProbability": 0.998964, "logPopularity": 6.013715156042802, "score": 0.9796745567601535, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Goto", "cookedLabel": "Goto", "pageID": "23307350", "editDist": 0.0, "labelProbability": 0.233129, "logPopularity": 3.258096538021482, "score": 0.06728267116099904, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "High school", "cookedLabel": "High school", "pageID": "42556", "editDist": 0.0, "labelProbability": 0.710763, "logPopularity": 8.718990524710849, "score": 0.9599863160325811, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "high school?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002280", "qText": "where are the headquarters of the united nations organization found?", "SV": ["found"], "lemmaSV": ["find"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "List of districts of India", "cookedLabel": "headquarters", "pageID": "602648", "editDist": 0.0, "labelProbability": 0.0687356, "logPopularity": 9.829248802600596, "score": 0.6382891701153312, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Headquarters", "cookedLabel": "Headquarters", "pageID": "745008", "editDist": 0.0, "labelProbability": 0.515517, "logPopularity": 4.174387269895637, "score": 0.5534525447014913, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Headquarters (album)", "cookedLabel": "Headquarters", "pageID": "2098872", "editDist": 0.0, "labelProbability": 0.199538, "logPopularity": 4.330733340286331, "score": 0.1062274079951419, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "United Nations", "cookedLabel": "United Nations", "pageID": "31769", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 5.66988092298052, "score": 0.9357004755501552, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Found (band)", "cookedLabel": "Found", "pageID": "13820845", "editDist": 0.0, "labelProbability": 0.439739, "logPopularity": 4.02535169073515, "score": 0.2280822046564627, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Found (novel)", "cookedLabel": "Found", "pageID": "20797033", "editDist": 0.0, "labelProbability": 0.439739, "logPopularity": 3.970291913552122, "score": 0.22231829380182722, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Found Aircraft", "cookedLabel": "Found Aircraft", "pageID": "9649134", "editDist": 0.0, "labelProbability": 0.439739, "logPopularity": 3.828641396489095, "score": 0.20797155950833424, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Found (Rossetti)", "cookedLabel": "Found", "pageID": "34500037", "editDist": 0.0, "labelProbability": 0.439739, "logPopularity": 3.4339872044851463, "score": 0.1716491629243687, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Found (album)", "cookedLabel": "Found", "pageID": "38290880", "editDist": 0.0, "labelProbability": 0.439739, "logPopularity": 3.9889840465642745, "score": 0.22426333707451532, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the headquarters of the united nations organization", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "headquarters of the united nations organization", "type": "CluePhrase", "weight": 0.99}, {"label": "united nations organization", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002300", "qText": "who was dr seuss?", "SV": [], "lemmaSV": [], "LAT": [{"text": "seuss", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Dr. Seuss", "cookedLabel": "Dr. Seuss", "pageID": "8855", "editDist": 0.0, "labelProbability": 0.910811, "logPopularity": 5.805134968916488, "score": 0.8715313449939921, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "dr seuss", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr002320", "qText": "who won 2001 fa cup?", "SV": ["won"], "lemmaSV": ["win"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "2010–11 FA Cup", "cookedLabel": "2010–11 FA Cup", "pageID": "27876486", "editDist": 1.5, "labelProbability": 0.0, "logPopularity": 7.076653815443951, "score": 0.2752145800797534, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "2001", "cookedLabel": "2001", "pageID": "34551", "editDist": 0.0, "labelProbability": 0.600947, "logPopularity": 2.833213344056216, "score": 0.2326426036402682, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "FA Cup", "cookedLabel": "FA Cup", "pageID": "11237", "editDist": 0.0, "labelProbability": 0.818182, "logPopularity": 7.216709486709457, "score": 0.9410383616804073, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "fa cup?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002340", "qText": "what was eli whitney job?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "6406508", "text": "book", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10344679", "text": "hero", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "783339", "text": "robbery", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "720746", "text": "duty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "576778", "text": "work", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4014270", "text": "product", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9646208", "text": "leader", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6365164", "text": "coding system", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "1082290", "text": "group action", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6367301", "text": "code", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4609402", "text": "workplace", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6581154", "text": "program", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13943868", "text": "condition", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6371284", "text": "writing", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "770190", "text": "felony", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "734044", "text": "wrongdoing", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "8637636", "text": "point", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "746303", "text": "transgression", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "1132241", "text": "duty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14431490", "text": "difficulty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "767761", "text": "crime", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "6578068", "text": "software", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "1125919", "text": "social control", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3133774", "text": "creation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24900", "text": "state", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "782543", "text": "larceny", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6374360", "text": "writing", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "767587", "text": "offense", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8596234", "text": "geographic point", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6582286", "text": "application", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9653829", "text": "unfortunate", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6403644", "text": "section", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6360590", "text": "written communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "job", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Eli Whitney", "cookedLabel": "Eli Whitney", "pageID": "9732", "editDist": 2.2, "labelProbability": 0.0, "logPopularity": 4.442651256490317, "score": 0.05922375965079095, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Eli Whitney", "cookedLabel": "Eli Whitney", "pageID": "9732", "editDist": 0.0, "labelProbability": 0.961735, "logPopularity": 4.442651256490317, "score": 0.8069882144943348, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "eli whitney job", "type": "ClueNE", "weight": 2.8000000000000003}, {"label": "eli whitney", "type": "ClueNgram", "weight": 1.01}] }, +{"qId": "wqr002360", "qText": "what football teams did emmitt smith play for?", "SV": ["play"], "lemmaSV": ["play"], "LAT": [{"synset": "7957410", "text": "biological group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8010371", "text": "animal group", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "teams", "specificity": "0.0", "type": "LAT"}, {"text": "team", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Emmitt Smith", "cookedLabel": "Emmitt Smith", "pageID": "154857", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.990432586778736, "score": 0.9632491468449639, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "football", "type": "ClueToken", "weight": 1.0}] }, +{"qId": "wqr002380", "qText": "where was karl marx buried?", "SV": ["buried"], "lemmaSV": ["bury"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Karl Marx", "cookedLabel": "Karl Marx", "pageID": "16743", "editDist": 0.0, "labelProbability": 0.964462, "logPopularity": 6.3473892096560105, "score": 0.9804868241857398, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002400", "qText": "who portrayed indiana jones in raiders of the lost ark?", "SV": ["portrayed"], "lemmaSV": ["portray"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Indiana Jones", "cookedLabel": "Indiana Jones", "pageID": "14814", "editDist": 0.0, "labelProbability": 0.868808, "logPopularity": 5.056245805348308, "score": 0.8463875964133596, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Indiana Jones (franchise)", "cookedLabel": "Indiana Jones", "pageID": "11903589", "editDist": 0.0, "labelProbability": 0.053644, "logPopularity": 4.6443908991413725, "score": 0.036496037661396225, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Raiders of the Lost Ark", "cookedLabel": "Raiders of the Lost Ark", "pageID": "54166", "editDist": 0.0, "labelProbability": 0.921928, "logPopularity": 4.700480365792417, "score": 0.8503381357740963, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Raiders of the Lost Ark (video game)", "cookedLabel": "Raiders of the Lost Ark", "pageID": "3490208", "editDist": 0.0, "labelProbability": 0.0511881, "logPopularity": 3.8066624897703196, "score": 0.022154971584016563, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002420", "qText": "where do islamic people go to worship?", "SV": ["go"], "lemmaSV": ["go"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Muslim", "cookedLabel": "Muslim", "pageID": "19541", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 6.661854740545311, "score": 0.41274687433602575, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Goto", "cookedLabel": "Goto", "pageID": "23307350", "editDist": 0.0, "labelProbability": 0.233129, "logPopularity": 3.258096538021482, "score": 0.06728267116099904, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Worship", "cookedLabel": "Worship", "pageID": "70364", "editDist": 0.0, "labelProbability": 0.179385, "logPopularity": 3.6635616461296463, "score": 0.06704711255854262, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Worship in Hinduism", "cookedLabel": "Worship in Hinduism", "pageID": "27185976", "editDist": 0.0, "labelProbability": 0.197608, "logPopularity": 1.0986122886681098, "score": 0.006218142222364109, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Christian worship", "cookedLabel": "Christian worship", "pageID": "1680963", "editDist": 0.0, "labelProbability": 0.232346, "logPopularity": 2.833213344056216, "score": 0.02035962494161789, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "islamic people", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr002440", "qText": "who the voice of peter griffin?", "SV": ["griffin"], "lemmaSV": ["griffin"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Who the *$&% Is Jackson Pollock?", "cookedLabel": "Who the *$&% Is Jackson Pollock?", "pageID": "8326198", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.9512437185814275, "score": 0.03522960933689275, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Voice of Eye", "cookedLabel": "Voice of Eye", "pageID": "18208561", "editDist": 3.0, "labelProbability": 0.0, "logPopularity": 3.044522437723423, "score": 0.05112070357256317, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Peter Griffin", "cookedLabel": "Peter Griffin", "pageID": "901191", "editDist": 0.0, "labelProbability": 0.965579, "logPopularity": 4.30406509320417, "score": 0.7965835370212142, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "voice of peter", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002460", "qText": "what was the capital of ancient israel?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "4606723", "text": "work", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8531106", "text": "center", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8067430", "text": "government", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8647614", "text": "region", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6422547", "text": "book", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6819327", "text": "symbol", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8681598", "text": "top", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8540894", "text": "center", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3133774", "text": "creation", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8514304", "text": "area", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6831828", "text": "character", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8069301", "text": "federal government", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13350663", "text": "assets", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8067137", "text": "polity", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8530790", "text": "place", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6601855", "text": "publication", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "32912", "text": "possession", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4014270", "text": "product", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "6830481", "text": "written symbol", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8665520", "text": "seat", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6804229", "text": "signal", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"text": "capital", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "The Capital", "cookedLabel": "The Capital", "pageID": "3257048", "editDist": 0.0, "labelProbability": 0.946108, "logPopularity": 3.713572066704308, "score": 0.6193803152222629, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "History of ancient Israel and Judah", "cookedLabel": "History of ancient Israel and Judah", "pageID": "13876", "editDist": 0.0, "labelProbability": 0.972222, "logPopularity": 3.5263605246161616, "score": 0.7797092703968829, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the capital of ancient israel", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "capital of ancient israel", "type": "CluePhrase", "weight": 0.99}, {"label": "ancient israel", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002480", "qText": "what time zone is ontario toronto?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9408804", "text": "part", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "2452", "text": "thing", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5227735", "text": "body part", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8647614", "text": "region", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8637636", "text": "point", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8682181", "text": "topographic point", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5232895", "text": "structure", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "zone", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Time zone", "cookedLabel": "Time zone", "pageID": "30890", "editDist": 0.0, "labelProbability": 0.695076, "logPopularity": 5.755742213586912, "score": 0.3922798159673489, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Toronto City Hall", "cookedLabel": "Toronto City Hall", "pageID": "798097", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.356708826689592, "score": 0.827075281260114, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "ontario toronto", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr002500", "qText": "who does zach galifianakis play in the hangover?", "SV": ["play"], "lemmaSV": ["play"], "LAT": [{"text": "galifianakis", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Zach Galifianakis", "cookedLabel": "Zach Galifianakis", "pageID": "644431", "editDist": 0.0, "labelProbability": 0.99935, "logPopularity": 5.087596335232384, "score": 0.8688719133299732, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Hangover", "cookedLabel": "The Hangover", "pageID": "21918632", "editDist": 0.0, "labelProbability": 0.509538, "logPopularity": 4.941642422609304, "score": 0.2690380695668368, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Hangover", "cookedLabel": "Hangover", "pageID": "12274183", "editDist": 0.0, "labelProbability": 0.480615, "logPopularity": 3.258096538021482, "score": 0.2392757898144611, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Hangover", "cookedLabel": "Hangover", "pageID": "12274183", "editDist": 0.0, "labelProbability": 0.480615, "logPopularity": 3.258096538021482, "score": 0.18367718907139213, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "hangover", "type": "ClueNE", "weight": 1.11}, {"label": "hangover?", "type": "ClueNgram", "weight": 1.01}] }, +{"qId": "wqr002520", "qText": "what was gregor mendel known for?", "SV": ["known"], "lemmaSV": ["know"], "LAT": [], "Concept": [{"fullLabel": "Gregor Mendel", "cookedLabel": "Gregor Mendel", "pageID": "12562", "editDist": 0.0, "labelProbability": 0.994921, "logPopularity": 4.68213122712422, "score": 0.9551192662824227, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002540", "qText": "what inspired monet?", "SV": ["inspired"], "lemmaSV": ["inspire"], "LAT": [], "Concept": [{"fullLabel": "Claude Monet", "cookedLabel": "Claude Monet", "pageID": "6548", "editDist": 0.0, "labelProbability": 0.98304, "logPopularity": 5.093750200806762, "score": 0.872042328775178, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "monet", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002560", "qText": "what is the actual current local time now in uk?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "5097645", "text": "magnitude", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4923519", "text": "property", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "15137796", "text": "time period", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5824748", "text": "datum", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "15269461", "text": "moment", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4990371", "text": "sound property", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "15205381", "text": "point", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5824916", "text": "reading", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "15249282", "text": "term", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7300108", "text": "experience", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13597072", "text": "fundamental quantity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7298313", "text": "happening", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4998633", "text": "rhythmicity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7323507", "text": "case", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5824413", "text": "information", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5100843", "text": "dimension", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "time", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "The Actual (novel)", "cookedLabel": "The Actual", "pageID": "9271125", "editDist": 0.0, "labelProbability": 0.43609, "logPopularity": 3.9512437185814275, "score": 0.2174806877733024, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Actual (band)", "cookedLabel": "The Actual", "pageID": "2317012", "editDist": 0.0, "labelProbability": 0.43609, "logPopularity": 3.784189633918261, "score": 0.2009063673527151, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Time zone", "cookedLabel": "Time zone", "pageID": "30890", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 5.755742213586912, "score": 0.06527190474495301, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "The Rolling Stones, Now!", "cookedLabel": "The Rolling Stones, Now!", "pageID": "1365981", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.543294782270004, "score": 0.13527332344325466, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (Fireflight album)", "cookedLabel": "Now", "pageID": "34423465", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.343805421853684, "score": 0.1218734455168399, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now! (Bobby Hutcherson album)", "cookedLabel": "Now!", "pageID": "2580911", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.248495242049359, "score": 0.1158846124007123, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now TV", "cookedLabel": "Now TV", "pageID": "2857006", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.430816798843313, "score": 0.1275717260871764, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (newspaper)", "cookedLabel": "Now", "pageID": "1058750", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.532599493153256, "score": 0.1345244471522166, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "United Kingdom", "cookedLabel": "United Kingdom", "pageID": "31717", "editDist": 0.0, "labelProbability": 0.514067, "logPopularity": 11.570967932364097, "score": 0.9746614553298311, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "UK (band)", "cookedLabel": "UK", "pageID": "925525", "editDist": 0.0, "labelProbability": 0.153453, "logPopularity": 4.836281906951478, "score": 0.04589382688062347, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Ukrainian language", "cookedLabel": "Ukrainian language", "pageID": "46279", "editDist": 0.0, "labelProbability": 0.153453, "logPopularity": 5.963579343618446, "score": 0.08642580612114378, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "University of Kentucky", "cookedLabel": "University of Kentucky", "pageID": "284368", "editDist": 0.0, "labelProbability": 0.153453, "logPopularity": 6.313548046277095, "score": 0.10450881959633042, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Uttarakhand", "cookedLabel": "Uttarakhand", "pageID": "1429154", "editDist": 0.0, "labelProbability": 0.153453, "logPopularity": 6.959398512133975, "score": 0.1467154939445143, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "the actual current local time now in uk", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "actual current local time now in uk", "type": "CluePhrase", "weight": 0.99}, {"label": "actual current local time now", "type": "CluePhrase", "weight": 0.99}, {"label": "actual current local time", "type": "CluePhrase", "weight": 0.99}, {"label": "current", "type": "ClueToken", "weight": 1.0}, {"label": "uk", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002580", "qText": "what are the four harry potter house names?", "SV": [], "lemmaSV": [], "LAT": [], "Concept": [{"fullLabel": "The Four (TV series)", "cookedLabel": "The Four", "pageID": "18817858", "editDist": 0.0, "labelProbability": 0.417476, "logPopularity": 4.465908118654584, "score": 0.25784972037557835, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Four (film)", "cookedLabel": "The Four", "pageID": "36856510", "editDist": 0.0, "labelProbability": 0.216828, "logPopularity": 3.8066624897703196, "score": 0.08509955283778954, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "List of Forgotten Realms characters", "cookedLabel": "the four", "pageID": "4308754", "editDist": 0.0, "labelProbability": 0.216828, "logPopularity": 4.927253685157205, "score": 0.15411933501456931, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Harry Potter", "cookedLabel": "Harry Potter", "pageID": "2387806", "editDist": 0.0, "labelProbability": 0.694104, "logPopularity": 4.762173934797756, "score": 0.5967886135480428, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Ephraim B. Potter House", "cookedLabel": "Ephraim B. Potter House", "pageID": "28900394", "editDist": 0.0, "labelProbability": 0.615385, "logPopularity": 4.007333185232471, "score": 0.3958885384021526, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Potter House (St. Petersburg, Florida)", "cookedLabel": "Potter House", "pageID": "7396470", "editDist": 0.0, "labelProbability": 0.615385, "logPopularity": 3.9318256327243257, "score": 0.38510636851009705, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Skene Manor", "cookedLabel": "Skene Manor", "pageID": "27689277", "editDist": 0.0, "labelProbability": 0.615385, "logPopularity": 3.970291913552122, "score": 0.39058583138487807, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Potter House (Rock Island, Illinois)", "cookedLabel": "Potter House", "pageID": "31342420", "editDist": 0.0, "labelProbability": 0.615385, "logPopularity": 3.9889840465642745, "score": 0.3932585908145613, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Arnold Potter House", "cookedLabel": "Arnold Potter House", "pageID": "24991909", "editDist": 0.0, "labelProbability": 0.615385, "logPopularity": 3.9889840465642745, "score": 0.3932585908145613, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the four harry potter house names", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "four harry potter house names", "type": "CluePhrase", "weight": 0.99}, {"label": "potter house names", "type": "CluePhrase", "weight": 0.99}, {"label": "potter house", "type": "ClueNE", "weight": 1.11}, {"label": "names", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr002600", "qText": "what drugs were in whitney houston when she died?", "SV": ["died"], "lemmaSV": ["die"], "LAT": [{"synset": "14802595", "text": "agent", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "20270", "text": "substance", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7347", "text": "causal agent", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "drugs", "specificity": "0.0", "type": "LAT"}, {"text": "drug", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Whitney Houston", "cookedLabel": "Whitney Houston", "pageID": "34071", "editDist": 0.0, "labelProbability": 0.96676, "logPopularity": 6.267200548541362, "score": 0.9470167387264969, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "She Died", "cookedLabel": "She Died", "pageID": "42580221", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.1354942159291497, "score": 0.02189317471715583, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Death", "cookedLabel": "Death", "pageID": "8221", "editDist": 0.0, "labelProbability": 0.32493, "logPopularity": 4.90527477843843, "score": 0.22812175118227904, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "whitney houston when she died", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "died?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002620", "qText": "where can you buy amazon kindle?", "SV": ["buy"], "lemmaSV": ["buy"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "You", "cookedLabel": "You", "pageID": "464907", "editDist": 0.0, "labelProbability": 0.166744, "logPopularity": 3.332204510175204, "score": 0.05265639583854093, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "You (Juju album)", "cookedLabel": "You", "pageID": "32465927", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.584967478670572, "score": 0.030978008205098034, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "To Know That You're Alive", "cookedLabel": "To Know That You're Alive", "pageID": "16113542", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.543294782270004, "score": 0.03023619203088639, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "You County", "cookedLabel": "You County", "pageID": "24702306", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.477336814478207, "score": 0.029097127371004054, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "You (Ten Sharp song)", "cookedLabel": "You", "pageID": "18041571", "editDist": 0.0, "labelProbability": 0.0973713, "logPopularity": 4.454347296253507, "score": 0.028709976328474045, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Amazon Kindle", "cookedLabel": "Amazon Kindle", "pageID": "14312829", "editDist": 0.0, "labelProbability": 0.99779, "logPopularity": 5.0106352940962555, "score": 0.9065351855269641, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "you", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr002640", "qText": "what type of poetry does john donne write?", "SV": ["write"], "lemmaSV": ["write"], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7957410", "text": "biological group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "2855782", "text": "block", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847533", "text": "kind", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6830481", "text": "written symbol", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6819327", "text": "symbol", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "6831828", "text": "character", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6804229", "text": "signal", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9628463", "text": "adult", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8008892", "text": "taxonomic group", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "type", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Poetry", "cookedLabel": "Poetry", "pageID": "22926", "editDist": 0.0, "labelProbability": 0.39542, "logPopularity": 7.283448228756631, "score": 0.6299349710349235, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Poetry (magazine)", "cookedLabel": "Poetry", "pageID": "1088973", "editDist": 0.0, "labelProbability": 0.0758634, "logPopularity": 4.499809670330265, "score": 0.02677974572143383, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Buddhist poetry", "cookedLabel": "Buddhist poetry", "pageID": "20337701", "editDist": 0.0, "labelProbability": 0.0549195, "logPopularity": 2.995732273553991, "score": 0.010034279784427006, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "John Donne", "cookedLabel": "John Donne", "pageID": "15838", "editDist": 0.0, "labelProbability": 0.988386, "logPopularity": 4.969813299576001, "score": 0.9608491601426676, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "poetry", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002660", "qText": "what does a american rottweiler look like?", "SV": ["look"], "lemmaSV": ["look"], "LAT": [{"synset": "1320032", "text": "domestic animal", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "2086723", "text": "dog", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "1474323", "text": "vertebrate", "specificity": "-8.0", "type": "WordnetLAT"}, {"synset": "2077948", "text": "carnivore", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "2085998", "text": "canine", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "2106058", "text": "working dog", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "15568", "text": "animal", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "1864419", "text": "mammal", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "1468898", "text": "chordate", "specificity": "-9.0", "type": "WordnetLAT"}, {"synset": "2107175", "text": "shepherd dog", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "1889397", "text": "placental", "specificity": "-6.0", "type": "WordnetLAT"}, {"text": "rottweiler", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Goon Affiliated", "cookedLabel": "Goon Affiliated", "pageID": "23030040", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.969813299576001, "score": 0.0859675096458276, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "a american rottweiler", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "american rottweiler", "type": "CluePhrase", "weight": 0.99}, {"label": "american", "type": "ClueToken", "weight": 1.0}, {"label": "rottweiler", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "look like?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002680", "qText": "what is president abraham lincoln known for?", "SV": ["known"], "lemmaSV": ["know"], "LAT": [], "Concept": [{"fullLabel": "Abraham Lincoln", "cookedLabel": "Abraham Lincoln", "pageID": "307", "editDist": 0.0, "labelProbability": 0.983871, "logPopularity": 6.089044875446846, "score": 0.9791874922326462, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "president abraham lincoln", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr002700", "qText": "what currency is used in hungary?", "SV": ["used"], "lemmaSV": ["use"], "LAT": [{"synset": "4923519", "text": "property", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13394134", "text": "medium of exchange", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7275291", "text": "standard", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4772610", "text": "prevalence", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5051824", "text": "temporal arrangement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5051679", "text": "temporal property", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4771667", "text": "generality", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13598374", "text": "system of measurement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5057266", "text": "presentness", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5053160", "text": "timing", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "currency", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Hungary", "cookedLabel": "Hungary", "pageID": "13275", "editDist": 0.0, "labelProbability": 0.578907, "logPopularity": 9.512369038134885, "score": 0.9377697837574935, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Hungary national football team", "cookedLabel": "Hungary national football team", "pageID": "679739", "editDist": 0.0, "labelProbability": 0.0682321, "logPopularity": 7.562681246721884, "score": 0.1430294998897364, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "hungary", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002720", "qText": "what college did joe montana play for?", "SV": ["play"], "lemmaSV": ["play"], "LAT": [{"synset": "8070328", "text": "institution", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8293263", "text": "educational institution", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7981699", "text": "body", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4348764", "text": "structure", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "2918337", "text": "building complex", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"text": "college", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Joe Montana", "cookedLabel": "Joe Montana", "pageID": "295701", "editDist": 0.0, "labelProbability": 0.97873, "logPopularity": 4.8283137373023015, "score": 0.9556853572424215, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002740", "qText": "who plays alan parrish in jumanji?", "SV": ["plays"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Alan Parrish", "cookedLabel": "Alan Parrish", "pageID": "5121242", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 1.9459101490553132, "score": 0.015094647817577586, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Jumanji", "cookedLabel": "Jumanji", "pageID": "3700174", "editDist": 0.0, "labelProbability": 0.77739, "logPopularity": 4.919980925828125, "score": 0.7046642431771835, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002760", "qText": "where did the arizona diamondbacks play?", "SV": ["play"], "lemmaSV": ["play"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Arizona Diamondbacks", "cookedLabel": "Arizona Diamondbacks", "pageID": "2129", "editDist": 0.0, "labelProbability": 0.927129, "logPopularity": 6.6039438246004725, "score": 0.9801458454473052, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002780", "qText": "where are riddell helmets manufactured?", "SV": ["manufactured"], "lemmaSV": ["manufacture"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [], "Clue": [{"label": "riddell helmets", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "riddell", "type": "ClueToken", "weight": 1.0}, {"label": "helmets", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr002800", "qText": "what movies did matt bomer play in?", "SV": ["play"], "lemmaSV": ["play"], "LAT": [{"synset": "4014270", "text": "product", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6631572", "text": "show", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3133774", "text": "creation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7303344", "text": "social event", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "movies", "specificity": "0.0", "type": "LAT"}, {"text": "movie", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Matt Bomer", "cookedLabel": "Matt Bomer", "pageID": "916180", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.276666119016055, "score": 0.944689765954643, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002820", "qText": "what did roger sherman do for a living?", "SV": ["do"], "lemmaSV": ["do"], "LAT": [{"synset": "9794206", "text": "advocate", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9759416", "text": "American Revolutionary leader", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9962718", "text": "commissioned military officer", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10143381", "text": "general", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9655706", "text": "worker", "specificity": "-8.0", "type": "WordnetLAT"}, {"synset": "9962449", "text": "commissioned officer", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8643858", "text": "municipality", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8683242", "text": "town", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10625393", "text": "skilled worker", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8637636", "text": "point", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8635538", "text": "peak", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10365746", "text": "nationalist", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10145323", "text": "general officer", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10365929", "text": "nationalist leader", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8693705", "text": "urban area", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10336665", "text": "military officer", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "10602198", "text": "serviceman", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "9383019", "text": "mountain peak", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9646208", "text": "leader", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8682181", "text": "topographic point", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-5.0", "type": "WordnetLAT"}, {"text": "sherman", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Roger Sherman", "cookedLabel": "Roger Sherman", "pageID": "260910", "editDist": 0.0, "labelProbability": 0.983931, "logPopularity": 4.955827057601261, "score": 0.8508251533681768, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "For a Living", "cookedLabel": "For a Living", "pageID": "14785803", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 3.8066624897703196, "score": 0.8263293338112999, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "for a living?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr002840", "qText": "where is christina aguilera from?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Christina Aguilera", "cookedLabel": "Christina Aguilera", "pageID": "144171", "editDist": 0.0, "labelProbability": 0.939469, "logPopularity": 6.1779441140506, "score": 0.9758827188302984, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Christina Aguilera (album)", "cookedLabel": "Christina Aguilera", "pageID": "555339", "editDist": 0.0, "labelProbability": 0.0512755, "logPopularity": 5.278114659230517, "score": 0.1292098233400038, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002860", "qText": "what is the dominant language of jamaica?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5658174", "text": "faculty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6297048", "text": "word", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6294878", "text": "language unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5624029", "text": "ability", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6399623", "text": "text", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13831419", "text": "part", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5778661", "text": "higher cognitive process", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6376912", "text": "matter", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "language", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Linguistic imperialism", "cookedLabel": "Linguistic imperialism", "pageID": "253283", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.833213344056216, "score": 0.016623034801995763, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Jamaica", "cookedLabel": "Jamaica", "pageID": "15660", "editDist": 0.0, "labelProbability": 0.69147, "logPopularity": 8.129764445794171, "score": 0.9168663449805127, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the dominant language of jamaica", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "dominant language of jamaica", "type": "CluePhrase", "weight": 0.99}, {"label": "dominant language", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr002880", "qText": "where was anne frank born?", "SV": ["born"], "lemmaSV": ["bear"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Anne Frank", "cookedLabel": "Anne Frank", "pageID": "804581", "editDist": 0.0, "labelProbability": 0.987277, "logPopularity": 5.056245805348308, "score": 0.9625708043797679, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002900", "qText": "what timezone is minnesota in?", "SV": [], "lemmaSV": [], "LAT": [{"text": "timezone", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Time zone", "cookedLabel": "Time zone", "pageID": "30890", "editDist": 0.0, "labelProbability": 0.797753, "logPopularity": 5.755742213586912, "score": 0.7369710831675726, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Minnesota", "cookedLabel": "Minnesota", "pageID": "19590", "editDist": 0.0, "labelProbability": 0.747924, "logPopularity": 9.432122997651051, "score": 0.988312426796726, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "timezone", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr002920", "qText": "where does frida kahlo live now?", "SV": ["live"], "lemmaSV": ["live"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Frida Kahlo", "cookedLabel": "Frida Kahlo", "pageID": "162276", "editDist": 0.0, "labelProbability": 0.995562, "logPopularity": 4.74493212836325, "score": 0.9568289214003222, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Rolling Stones, Now!", "cookedLabel": "The Rolling Stones, Now!", "pageID": "1365981", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.543294782270004, "score": 0.13527332344325466, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now TV", "cookedLabel": "Now TV", "pageID": "2857006", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.430816798843313, "score": 0.1275717260871764, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now! (Bobby Hutcherson album)", "cookedLabel": "Now!", "pageID": "2580911", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.248495242049359, "score": 0.1158846124007123, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (Fireflight album)", "cookedLabel": "Now", "pageID": "34423465", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.343805421853684, "score": 0.1218734455168399, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Now (newspaper)", "cookedLabel": "Now", "pageID": "1058750", "editDist": 0.0, "labelProbability": 0.233777, "logPopularity": 4.532599493153256, "score": 0.1345244471522166, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "now", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr002940", "qText": "where does the jordan river end?", "SV": ["end"], "lemmaSV": ["end"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Jordan River", "cookedLabel": "Jordan River", "pageID": "47910", "editDist": 0.0, "labelProbability": 0.845543, "logPopularity": 4.74493212836325, "score": 0.9175028154365124, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Jordan River (Utah)", "cookedLabel": "Jordan River", "pageID": "480458", "editDist": 0.0, "labelProbability": 0.117741, "logPopularity": 4.672828834461906, "score": 0.12286016435052517, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr002960", "qText": "who was papa doc in real life?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "8139116", "text": "federal department", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-10.0", "type": "WordnetLAT"}, {"synset": "8237635", "text": "division", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8094128", "text": "administrative unit", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "10184702", "text": "health professional", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-8.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "8140150", "text": "executive department", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8136796", "text": "government department", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10325469", "text": "medical practitioner", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10499838", "text": "professional", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9628463", "text": "adult", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-9.0", "type": "WordnetLAT"}, {"synset": "8131836", "text": "department", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "doc", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "François Duvalier", "cookedLabel": "François Duvalier", "pageID": "70844", "editDist": 0.0, "labelProbability": 0.999273, "logPopularity": 4.969813299576001, "score": 0.8153300249179136, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Real life", "cookedLabel": "Real life", "pageID": "238114", "editDist": 0.0, "labelProbability": 0.835073, "logPopularity": 3.091042453358316, "score": 0.5920343108437078, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "In Real Life", "cookedLabel": "In Real Life", "pageID": "5313706", "editDist": 0.0, "labelProbability": 0.164927, "logPopularity": 3.8066624897703196, "score": 0.09291252556801953, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "papa doc in real life", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "papa doc", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr002980", "qText": "where do most earthquakes happen in japan?", "SV": ["happen"], "lemmaSV": ["happen"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "In Japan! (Buck Owens album)", "cookedLabel": "In Japan!", "pageID": "19866078", "editDist": 0.0, "labelProbability": 0.618506, "logPopularity": 3.8501476017100584, "score": 0.45819433935440645, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Jackson 5 in Japan", "cookedLabel": "The Jackson 5 in Japan", "pageID": "2160503", "editDist": 0.0, "labelProbability": 0.618506, "logPopularity": 3.6635616461296463, "score": 0.4305607973886944, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "In Japan (Mr. Big album)", "cookedLabel": "In Japan", "pageID": "983201", "editDist": 0.0, "labelProbability": 0.618506, "logPopularity": 3.713572066704308, "score": 0.43793234745286186, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "most earthquakes", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "most", "type": "ClueToken", "weight": 1.0}, {"label": "earthquakes", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "in japan?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003000", "qText": "what kind of language do china speak?", "SV": ["speak"], "lemmaSV": ["speak"], "LAT": [{"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "kind", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Language", "cookedLabel": "Language", "pageID": "17524", "editDist": 0.0, "labelProbability": 0.25619, "logPopularity": 5.493061443340548, "score": 0.23464873939729444, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "China", "cookedLabel": "China", "pageID": "5405", "editDist": 0.0, "labelProbability": 0.498325, "logPopularity": 10.34663372761198, "score": 0.9789357204017793, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr003020", "qText": "what language do irish people speak?", "SV": ["speak"], "lemmaSV": ["speak"], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5658174", "text": "faculty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6297048", "text": "word", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6294878", "text": "language unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5624029", "text": "ability", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6399623", "text": "text", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13831419", "text": "part", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5778661", "text": "higher cognitive process", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6376912", "text": "matter", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "language", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Irish people", "cookedLabel": "Irish people", "pageID": "775859", "editDist": 0.0, "labelProbability": 0.712121, "logPopularity": 7.537962659768208, "score": 0.9698631076524369, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "List of Irish people", "cookedLabel": "irish people", "pageID": "68826", "editDist": 0.0, "labelProbability": 0.136364, "logPopularity": 1.0986122886681098, "score": 0.01755838013404058, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "The Sunday People", "cookedLabel": "The Sunday People", "pageID": "689457", "editDist": 0.0, "labelProbability": 0.0757576, "logPopularity": 3.4339872044851463, "score": 0.052060329004864195, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "irish people", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr003040", "qText": "where do people speak burmese?", "SV": ["speak"], "lemmaSV": ["speak"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "People (magazine)", "cookedLabel": "People", "pageID": "507970", "editDist": 0.0, "labelProbability": 0.174827, "logPopularity": 4.584967478670572, "score": 0.10998291320708259, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "People", "cookedLabel": "People", "pageID": "3488351", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.0794415416798357, "score": 0.031159073502995693, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Burmese language", "cookedLabel": "Burmese language", "pageID": "338207", "editDist": 0.0, "labelProbability": 0.39045, "logPopularity": 5.736572297479192, "score": 0.39675240447609883, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Mon script", "cookedLabel": "Mon script", "pageID": "28769803", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.6375861597263857, "score": 0.02936365156468993, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Burmese python", "cookedLabel": "Burmese python", "pageID": "819149", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.8066624897703196, "score": 0.03239717786085767, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Burmese (horse)", "cookedLabel": "Burmese", "pageID": "4473528", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.9444389791664403, "score": 0.019568435709475668, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Burmese (cat)", "cookedLabel": "Burmese", "pageID": "261787", "editDist": 0.0, "labelProbability": 0.0608696, "logPopularity": 3.332204510175204, "score": 0.03303719981396769, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "burmese?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003060", "qText": "where does the band metallica live?", "SV": ["live"], "lemmaSV": ["live"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "The Band", "cookedLabel": "The Band", "pageID": "30965", "editDist": 0.0, "labelProbability": 0.194761, "logPopularity": 5.811140992976701, "score": 0.2811461575746737, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Musical ensemble", "cookedLabel": "Musical ensemble", "pageID": "20180", "editDist": 0.0, "labelProbability": 0.247375, "logPopularity": 4.5217885770490405, "score": 0.18686048882274314, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Rede Bandeirantes", "cookedLabel": "Rede Bandeirantes", "pageID": "7673602", "editDist": 0.0, "labelProbability": 0.194761, "logPopularity": 4.584967478670572, "score": 0.15782858823186155, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Band, Mureș", "cookedLabel": "Band, Mureș", "pageID": "12023460", "editDist": 0.0, "labelProbability": 0.194761, "logPopularity": 4.6913478822291435, "score": 0.16649881397835484, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The Band (album)", "cookedLabel": "The Band", "pageID": "197540", "editDist": 0.0, "labelProbability": 0.194761, "logPopularity": 4.736198448394496, "score": 0.17026686952351894, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the band metallica", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "band metallica", "type": "CluePhrase", "weight": 0.99}, {"label": "band", "type": "ClueNE", "weight": 1.11}, {"label": "metallica", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr003080", "qText": "when did the arab israeli war start?", "SV": ["start"], "lemmaSV": ["start"], "LAT": [{"synset": "15147173", "text": "time", "specificity": "0.0", "type": "QuestionWordLAT"}, {"synset": "15184543", "text": "date", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "1948 Arab–Israeli War", "cookedLabel": "1948 Arab–Israeli War", "pageID": "36197", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 5.552959584921617, "score": 0.9319954088284126, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Arab–Israeli conflict", "cookedLabel": "Arab–Israeli conflict", "pageID": "7960202", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.7535901911063645, "score": 0.18278984644239052, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "arab israeli war", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr003100", "qText": "what is the dominican republic part of?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "6005806", "text": "discipline", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5677778", "text": "cognitive state", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "3593583", "text": "item", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "720746", "text": "duty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "576778", "text": "work", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "2452", "text": "thing", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "32912", "text": "possession", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "787849", "text": "attempt", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "24900", "text": "state", "specificity": "-7.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8610818", "text": "line", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3553", "text": "whole", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13943868", "text": "condition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13350663", "text": "assets", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5690411", "text": "curiosity", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "14396987", "text": "psychological state", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5678554", "text": "concern", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "549839", "text": "portrayal", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6167042", "text": "performing arts", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "549363", "text": "acting", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6008444", "text": "knowledge domain", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-8.0", "type": "WordnetLAT"}, {"synset": "5690773", "text": "interest", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "27365", "text": "location", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6163352", "text": "humanistic discipline", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7041860", "text": "tune", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "2684", "text": "object", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "part", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Dominican Republic", "cookedLabel": "Dominican Republic", "pageID": "8060", "editDist": 0.0, "labelProbability": 0.671348, "logPopularity": 8.183676582620658, "score": 0.935555978041983, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the dominican republic part of", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "dominican republic part of", "type": "CluePhrase", "weight": 0.99}, {"label": "dominican republic part", "type": "CluePhrase", "weight": 0.99}, {"label": "part", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr003120", "qText": "who plays harold saxon in doctor who?", "SV": ["plays"], "lemmaSV": ["play"], "LAT": [{"synset": "10184702", "text": "health professional", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9644715", "text": "intellectual", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "427931", "text": "diversion", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10499838", "text": "professional", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "432833", "text": "play", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "408356", "text": "activity", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "10325469", "text": "medical practitioner", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10725264", "text": "theologian", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9628463", "text": "adult", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "10577282", "text": "scholar", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "doctor", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Story arcs in Doctor Who", "cookedLabel": "Story arcs in Doctor Who", "pageID": "5600732", "editDist": 0.0, "labelProbability": 0.5, "logPopularity": 3.5553480614894135, "score": 0.2912742714975, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Master (Doctor Who)", "cookedLabel": "Master", "pageID": "155600", "editDist": 0.0, "labelProbability": 0.469388, "logPopularity": 4.23410650459726, "score": 0.16678700446241362, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Doctor Who", "cookedLabel": "Doctor Who", "pageID": "8209", "editDist": 0.0, "labelProbability": 0.571429, "logPopularity": 6.124683390894205, "score": 0.3894211731510105, "getByLAT": 1, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "doctor who?", "type": "ClueNE", "weight": 1.6}] }, +{"qId": "wqr003140", "qText": "what kind of political system does iran have?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "kind", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Coal Run Village, Kentucky", "cookedLabel": "Coal Run Village, Kentucky", "pageID": "115405", "editDist": 0.0, "labelProbability": 0.142857, "logPopularity": 4.718498871295094, "score": 0.05631447464088684, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Political system", "cookedLabel": "Political system", "pageID": "258724", "editDist": 0.0, "labelProbability": 0.238095, "logPopularity": 2.1972245773362196, "score": 0.05176398693827355, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Politics of Germany", "cookedLabel": "Politics of Germany", "pageID": "11935", "editDist": 0.0, "labelProbability": 0.190476, "logPopularity": 4.1588830833596715, "score": 0.050415590449092426, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Iran", "cookedLabel": "Iran", "pageID": "14653", "editDist": 0.0, "labelProbability": 0.726237, "logPopularity": 11.200212722793202, "score": 0.995497558213468, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "political system", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003160", "qText": "what language do brazil speak?", "SV": ["speak"], "lemmaSV": ["speak"], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5658174", "text": "faculty", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6297048", "text": "word", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6294878", "text": "language unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5624029", "text": "ability", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6399623", "text": "text", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13831419", "text": "part", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5778661", "text": "higher cognitive process", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6376912", "text": "matter", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "language", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Brazil", "cookedLabel": "Brazil", "pageID": "3383", "editDist": 0.0, "labelProbability": 0.671435, "logPopularity": 10.453572350254236, "score": 0.9909755240376303, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Brazil national football team", "cookedLabel": "Brazil national football team", "pageID": "149286", "editDist": 0.0, "labelProbability": 0.0639685, "logPopularity": 7.789868559054706, "score": 0.336803569077327, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "brazil", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr003180", "qText": "what type of political system is spain?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7957410", "text": "biological group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "2855782", "text": "block", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847533", "text": "kind", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6830481", "text": "written symbol", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6819327", "text": "symbol", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "6831828", "text": "character", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6804229", "text": "signal", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9628463", "text": "adult", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8008892", "text": "taxonomic group", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "type", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Coal Run Village, Kentucky", "cookedLabel": "Coal Run Village, Kentucky", "pageID": "115405", "editDist": 0.0, "labelProbability": 0.142857, "logPopularity": 4.718498871295094, "score": 0.05631447464088684, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Political system", "cookedLabel": "Political system", "pageID": "258724", "editDist": 0.0, "labelProbability": 0.238095, "logPopularity": 2.1972245773362196, "score": 0.05176398693827355, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Politics of Germany", "cookedLabel": "Politics of Germany", "pageID": "11935", "editDist": 0.0, "labelProbability": 0.190476, "logPopularity": 4.1588830833596715, "score": 0.050415590449092426, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Spain", "cookedLabel": "Spain", "pageID": "26667", "editDist": 0.0, "labelProbability": 0.708413, "logPopularity": 10.559711378991475, "score": 0.9928424271025632, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "political system", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003200", "qText": "who is the 2011 heisman trophy winner?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "10138501", "text": "gambler", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9636221", "text": "contestant", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "winner", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Whois", "cookedLabel": "Whois", "pageID": "4315433", "editDist": 0.0, "labelProbability": 0.0673077, "logPopularity": 3.1780538303479458, "score": 0.031085894047283118, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "2011", "cookedLabel": "2011", "pageID": "36225", "editDist": 0.0, "labelProbability": 0.353935, "logPopularity": 2.6390573296152584, "score": 0.1901151086570081, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Heisman Trophy", "cookedLabel": "Heisman Trophy", "pageID": "288191", "editDist": 0.0, "labelProbability": 0.957449, "logPopularity": 3.6109179126442243, "score": 0.7133737462450445, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "who is", "type": "ClueNE", "weight": 1.11}, {"label": "the 2011 heisman trophy winner", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "2011 heisman trophy winner", "type": "CluePhrase", "weight": 0.99}, {"label": "winner", "type": "ClueSubjectToken", "weight": 2.5}] }, +{"qId": "wqr003220", "qText": "what is michael buble's style of music?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "5757616", "text": "taste", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "19308", "text": "natural object", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5477841", "text": "process", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5755999", "text": "discrimination", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "11696293", "text": "reproductive structure", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4459089", "text": "tool", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5709328", "text": "process", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3580409", "text": "instrumentality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "6611268", "text": "message", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13108385", "text": "plant organ", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5709891", "text": "basic cognitive process", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6799486", "text": "direction", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4819517", "text": "elegance", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13107668", "text": "plant part", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "3569147", "text": "implement", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5847533", "text": "kind", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4923519", "text": "property", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "style", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Michael Bublé", "cookedLabel": "Michael Bublé", "pageID": "621503", "editDist": 0.0, "labelProbability": 0.987684, "logPopularity": 5.641907070938114, "score": 0.931140929939333, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "michael buble's style of music", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "michael buble's style", "type": "CluePhrase", "weight": 0.99}, {"label": "michael buble", "type": "ClueNE", "weight": 1.11}, {"label": "style", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "music", "type": "ClueToken", "weight": 1.0}] }, +{"qId": "wqr003240", "qText": "what country did ponce de leon live in?", "SV": ["live"], "lemmaSV": ["live"], "LAT": [{"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7958392", "text": "people", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "country", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Juan Ponce de León", "cookedLabel": "Juan Ponce de León", "pageID": "143363", "editDist": 0.0, "labelProbability": 0.654321, "logPopularity": 4.8283137373023015, "score": 0.8291999303509269, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Daniel Ponce de León", "cookedLabel": "Daniel Ponce de León", "pageID": "6145533", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.406719247264253, "score": 0.15372514941192528, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Fernando Ponce de León", "cookedLabel": "Fernando Ponce de León", "pageID": "27239457", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.219507705176107, "score": 0.13967369476450198, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Nicolás Ponce de León", "cookedLabel": "Nicolás Ponce de León", "pageID": "27977311", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.248495242049359, "score": 0.1417767546521046, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Juan Ponce de León II", "cookedLabel": "Juan Ponce de León II", "pageID": "690491", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.454347296253507, "score": 0.15747964371107007, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "ponce de leon", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr003260", "qText": "what kind of government does vietnam have?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "kind", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Government", "cookedLabel": "Government", "pageID": "12229", "editDist": 0.0, "labelProbability": 0.139222, "logPopularity": 5.5093883366279774, "score": 0.15315169535105763, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Vietnam", "cookedLabel": "Vietnam", "pageID": "202354", "editDist": 0.0, "labelProbability": 0.661886, "logPopularity": 8.475120414994329, "score": 0.9697577991806455, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Vietnam national football team", "cookedLabel": "Vietnam national football team", "pageID": "1145542", "editDist": 0.0, "labelProbability": 0.0791402, "logPopularity": 6.386879319362645, "score": 0.19006117366778638, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Vietnam War", "cookedLabel": "Vietnam War", "pageID": "32611", "editDist": 0.0, "labelProbability": 0.0640296, "logPopularity": 8.210939733379021, "score": 0.3954040036925538, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "vietnam", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr003280", "qText": "who owns the steelers football team?", "SV": ["owns"], "lemmaSV": ["own"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Pittsburgh Steelers", "cookedLabel": "Pittsburgh Steelers", "pageID": "23338", "editDist": 0.0, "labelProbability": 0.875, "logPopularity": 7.58629630715272, "score": 0.9487306991039992, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Football team", "cookedLabel": "Football team", "pageID": "10830", "editDist": 0.0, "labelProbability": 0.2, "logPopularity": 2.833213344056216, "score": 0.04580817491713056, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "steelers football team", "type": "CluePhrase", "weight": 0.99}, {"label": "the steelers", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003300", "qText": "where is rihanna from ethnically?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Rihanna", "cookedLabel": "Rihanna", "pageID": "2110323", "editDist": 0.0, "labelProbability": 0.990786, "logPopularity": 6.236369590203704, "score": 0.9743285979093527, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Ethnic group", "cookedLabel": "Ethnic group", "pageID": "105004", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 5.19295685089021, "score": 0.8866146234875648, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "ethnically", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr003320", "qText": "who was the owner of kfc?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9901459", "text": "businessman", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9902168", "text": "businessperson", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "9632262", "text": "capitalist", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "owner", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Ownership", "cookedLabel": "Ownership", "pageID": "213897", "editDist": 0.0, "labelProbability": 0.858809, "logPopularity": 4.787491742782046, "score": 0.7436382102020456, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "List of professional sports team owners", "cookedLabel": "owner", "pageID": "7964652", "editDist": 0.0, "labelProbability": 0.111424, "logPopularity": 0.6931471805599453, "score": 0.0029799632449238897, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "KFC", "cookedLabel": "KFC", "pageID": "37404", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 4.219507705176107, "score": 0.8134460615072249, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the owner of kfc", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "owner of kfc", "type": "CluePhrase", "weight": 0.99}, {"label": "owner", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr003340", "qText": "what to do in chicago this weekend with kids?", "SV": ["do"], "lemmaSV": ["do"], "LAT": [], "Concept": [{"fullLabel": "OK Go (album)", "cookedLabel": "OK Go", "pageID": "2145907", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.532599493153256, "score": 0.0674698807859964, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Chicago", "cookedLabel": "Chicago", "pageID": "6886", "editDist": 0.0, "labelProbability": 0.760141, "logPopularity": 9.834887461043872, "score": 0.9767792272076851, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "This Weekend", "cookedLabel": "This Weekend", "pageID": "6458704", "editDist": 0.0, "labelProbability": 1.0, "logPopularity": 1.791759469228055, "score": 0.5868389547717006, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Kids (film)", "cookedLabel": "Kids", "pageID": "615418", "editDist": 0.0, "labelProbability": 0.395721, "logPopularity": 4.634728988229636, "score": 0.25809738507671115, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Kid Ory", "cookedLabel": "Kid Ory", "pageID": "97641", "editDist": 0.0, "labelProbability": 0.127217, "logPopularity": 4.787491742782046, "score": 0.09989393113722553, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Kid Gleason", "cookedLabel": "Kid Gleason", "pageID": "673658", "editDist": 0.0, "labelProbability": 0.127217, "logPopularity": 4.836281906951478, "score": 0.10255706926785661, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Kid Cudi", "cookedLabel": "Kid Cudi", "pageID": "19583036", "editDist": 0.0, "labelProbability": 0.127217, "logPopularity": 5.68697535633982, "score": 0.15993343942047092, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Kid Rock", "cookedLabel": "Kid Rock", "pageID": "17396", "editDist": 0.0, "labelProbability": 0.127217, "logPopularity": 6.042632833682381, "score": 0.19072088769920859, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "kids", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr003360", "qText": "what kind of legal system does australia have?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "kind", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Judicial system of Turkey", "cookedLabel": "Judicial system of Turkey", "pageID": "2963429", "editDist": 0.0, "labelProbability": 0.984375, "logPopularity": 3.044522437723423, "score": 0.5112052886391106, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "List of national legal systems", "cookedLabel": "legal system", "pageID": "154708", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.833213344056216, "score": 0.025435790265438976, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Australia", "cookedLabel": "Australia", "pageID": "4689264", "editDist": 0.0, "labelProbability": 0.747368, "logPopularity": 10.900768235614668, "score": 0.9951120776945014, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "legal system", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003380", "qText": "what currency do the ukraine use?", "SV": ["use"], "lemmaSV": ["use"], "LAT": [{"synset": "4923519", "text": "property", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13394134", "text": "medium of exchange", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7275291", "text": "standard", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4772610", "text": "prevalence", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5051824", "text": "temporal arrangement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5051679", "text": "temporal property", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4771667", "text": "generality", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13598374", "text": "system of measurement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5057266", "text": "presentness", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5053160", "text": "timing", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "currency", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Ukraine", "cookedLabel": "Ukraine", "pageID": "31750", "editDist": 0.0, "labelProbability": 0.607005, "logPopularity": 9.140239744296693, "score": 0.9811013965429601, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Ukrainian Soviet Socialist Republic", "cookedLabel": "Ukrainian Soviet Socialist Republic", "pageID": "376732", "editDist": 0.0, "labelProbability": 0.0990504, "logPopularity": 8.187021067343505, "score": 0.5142325907595652, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "ukraine", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr003400", "qText": "who plays stewie griffin on family guy?", "SV": ["plays"], "lemmaSV": ["play"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Stewie Griffin", "cookedLabel": "Stewie Griffin", "pageID": "530189", "editDist": 0.0, "labelProbability": 0.945701, "logPopularity": 4.0943445622221, "score": 0.8150031749808776, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Family Guy", "cookedLabel": "Family Guy", "pageID": "187586", "editDist": 0.0, "labelProbability": 0.964212, "logPopularity": 6.248042874508429, "score": 0.9258868087401481, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr003420", "qText": "which airport is closest to barcelona port?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "3319968", "text": "facility", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "2690851", "text": "airfield", "specificity": "-1.0", "type": "WordnetLAT"}, {"text": "airport", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Wang Yangming", "cookedLabel": "Wang Yangming", "pageID": "619526", "editDist": 0.0, "labelProbability": 0.571429, "logPopularity": 4.430816798843313, "score": 0.41085711020826166, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Boards of Canada", "cookedLabel": "Boards of Canada", "pageID": "101580", "editDist": 1.0, "labelProbability": 0.0, "logPopularity": 4.9344739331306915, "score": 0.1957567557938664, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "List of airports in South Dakota", "cookedLabel": "closest", "pageID": "5689688", "editDist": 0.0, "labelProbability": 0.142857, "logPopularity": 2.0794415416798357, "score": 0.023176803324641126, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "List of airports in Nebraska", "cookedLabel": "closest", "pageID": "5544226", "editDist": 0.0, "labelProbability": 0.142857, "logPopularity": 2.0794415416798357, "score": 0.023176803324641126, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Port of Barcelona", "cookedLabel": "Port of Barcelona", "pageID": "3526295", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.258096538021482, "score": 0.032580826260036944, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "closest to barcelona port", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "closest", "type": "ClueNE", "weight": 2.6}] }, +{"qId": "wqr003440", "qText": "what should i visit in venice?", "SV": ["visit"], "lemmaSV": ["visit"], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "14647071", "text": "chemical element", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "14928812", "text": "halogen", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "13597304", "text": "definite quantity", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "6831828", "text": "character", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "19793", "text": "substance", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13763162", "text": "digit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13750609", "text": "integer", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6819327", "text": "symbol", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "6830481", "text": "written symbol", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6841868", "text": "letter", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6804229", "text": "signal", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13603216", "text": "number", "specificity": "-3.0", "type": "WordnetLAT"}, {"text": "i", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Should I Stay or Should I Go", "cookedLabel": "Should I Stay or Should I Go", "pageID": "3629521", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.31748811353631, "score": 0.028619481308881617, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Venice", "cookedLabel": "Venice", "pageID": "32616", "editDist": 0.0, "labelProbability": 0.741286, "logPopularity": 7.216709486709457, "score": 0.8891086358339383, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Venice, Los Angeles", "cookedLabel": "Venice, Los Angeles", "pageID": "32579", "editDist": 0.0, "labelProbability": 0.0568927, "logPopularity": 5.883322388488279, "score": 0.054677647877007574, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Republic of Venice", "cookedLabel": "Republic of Venice", "pageID": "613492", "editDist": 0.0, "labelProbability": 0.074145, "logPopularity": 6.073044534100405, "score": 0.06556257763924893, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "venice", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr003460", "qText": "what country is beside france?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7958392", "text": "people", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "country", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Country Is", "cookedLabel": "Country Is", "pageID": "25433212", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 3.8501476017100584, "score": 0.008153680212262971, "getByLAT": 1, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "France", "cookedLabel": "France", "pageID": "5843419", "editDist": 3.0, "labelProbability": 0.0, "logPopularity": 11.423164857762606, "score": 0.95697806051839, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "beside france", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr003480", "qText": "who started mary kay?", "SV": ["started"], "lemmaSV": ["start"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Mary Kay", "cookedLabel": "Mary Kay", "pageID": "1583427", "editDist": 0.0, "labelProbability": 0.90873, "logPopularity": 4.110873864173311, "score": 0.7896527480944822, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr003500", "qText": "who was the voice of simba?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9794206", "text": "advocate", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4923519", "text": "property", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "7041860", "text": "tune", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10435383", "text": "performer", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9639952", "text": "entertainer", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "30657", "text": "act", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13819354", "text": "linguistic relation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7034009", "text": "music", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "44888", "text": "implementation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4990371", "text": "sound property", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7123727", "text": "auditory communication", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "7298313", "text": "happening", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "32220", "text": "relation", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "13818991", "text": "grammatical relation", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "173531", "text": "means", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5208927", "text": "physical ability", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "10619214", "text": "singer", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6262268", "text": "communication", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10360025", "text": "musician", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "4988388", "text": "sound", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7385893", "text": "sound", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7154581", "text": "expression", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5207437", "text": "ability", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "voice", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Simba", "cookedLabel": "Simba", "pageID": "983014", "editDist": 0.0, "labelProbability": 0.617504, "logPopularity": 4.31748811353631, "score": 0.44354266871782344, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Simba Makoni", "cookedLabel": "Simba Makoni", "pageID": "15257568", "editDist": 0.0, "labelProbability": 0.123501, "logPopularity": 4.6913478822291435, "score": 0.037002065590378067, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Simba Technologies", "cookedLabel": "Simba Technologies", "pageID": "9097838", "editDist": 0.0, "labelProbability": 0.123501, "logPopularity": 3.7612001156935624, "score": 0.021517087069021317, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Simba Rebellion", "cookedLabel": "Simba Rebellion", "pageID": "22017800", "editDist": 0.0, "labelProbability": 0.123501, "logPopularity": 4.276666119016055, "score": 0.029088793397246016, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Simba (film)", "cookedLabel": "Simba", "pageID": "27280272", "editDist": 0.0, "labelProbability": 0.123501, "logPopularity": 3.9889840465642745, "score": 0.024590718006657068, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "the voice of simba", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "voice of simba", "type": "CluePhrase", "weight": 0.99}, {"label": "voice", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "simba", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr003520", "qText": "where was the first ford motor company located?", "SV": ["located"], "lemmaSV": ["locate"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "The First (album)", "cookedLabel": "The First", "pageID": "33680093", "editDist": 0.0, "labelProbability": 0.270512, "logPopularity": 4.795790545596741, "score": 0.17729699269180282, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The First (musical)", "cookedLabel": "The First", "pageID": "8438202", "editDist": 0.0, "labelProbability": 0.270512, "logPopularity": 3.6375861597263857, "score": 0.09711679219405046, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "The First 48", "cookedLabel": "The First 48", "pageID": "9686259", "editDist": 0.0, "labelProbability": 0.270512, "logPopularity": 3.8501476017100584, "score": 0.1088885732049012, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Ford Motor Company", "cookedLabel": "Ford Motor Company", "pageID": "30433662", "editDist": 0.0, "labelProbability": 0.983606, "logPopularity": 7.491645473605133, "score": 0.990910423790686, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the first ford motor company", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "first ford motor company", "type": "CluePhrase", "weight": 0.99}, {"label": "the first", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003540", "qText": "when did the mets win the pennant?", "SV": ["win"], "lemmaSV": ["win"], "LAT": [{"synset": "15147173", "text": "time", "specificity": "0.0", "type": "QuestionWordLAT"}, {"synset": "15184543", "text": "date", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "New York Mets", "cookedLabel": "New York Mets", "pageID": "21728", "editDist": 0.0, "labelProbability": 0.529412, "logPopularity": 7.575071699507561, "score": 0.9104279922078052, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Jermaine Pennant", "cookedLabel": "Jermaine Pennant", "pageID": "1024810", "editDist": 0.0, "labelProbability": 0.199319, "logPopularity": 5.056245805348308, "score": 0.15371741856367757, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Pennant, Powys", "cookedLabel": "Pennant, Powys", "pageID": "30960035", "editDist": 0.0, "labelProbability": 0.203578, "logPopularity": 3.5553480614894135, "score": 0.07000041333163418, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Pennant Hills, New South Wales", "cookedLabel": "Pennant Hills, New South Wales", "pageID": "1110694", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 4.912654885736052, "score": 0.061044378088798555, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Pennant, Saskatchewan", "cookedLabel": "Pennant, Saskatchewan", "pageID": "15774015", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 5.19295685089021, "score": 0.07142594132589199, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Thomas Pennant", "cookedLabel": "Thomas Pennant", "pageID": "361699", "editDist": 0.0, "labelProbability": 0.183986, "logPopularity": 4.624972813284271, "score": 0.11557944742461398, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the mets", "type": "ClueNE", "weight": 2.8000000000000003}] }, +{"qId": "wqr003560", "qText": "who wrote 2 timothy 4?", "SV": ["wrote"], "lemmaSV": ["write"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Second Epistle to Timothy", "cookedLabel": "Second Epistle to Timothy", "pageID": "1751383", "editDist": 0.0, "labelProbability": 0.984968, "logPopularity": 3.58351893845611, "score": 0.7353383189937804, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Second Epistle to Timothy", "cookedLabel": "Second Epistle to Timothy", "pageID": "1751383", "editDist": 1.0, "labelProbability": 0.0, "logPopularity": 3.58351893845611, "score": 0.03843269416191313, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}], "Clue": [{"label": "2 timothy", "type": "ClueNgram", "weight": 1.01}, {"label": "timothy 4", "type": "ClueNE", "weight": 1.1}] }, +{"qId": "wqr003580", "qText": "where did fred west work?", "SV": ["work"], "lemmaSV": ["work"], "LAT": [{"synset": "27365", "text": "location", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Fred West", "cookedLabel": "Fred West", "pageID": "200822", "editDist": 0.0, "labelProbability": 0.99942, "logPopularity": 4.465908118654584, "score": 0.9502055963037747, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr003600", "qText": "what city was nelson mandela born in?", "SV": ["born"], "lemmaSV": ["bear"], "LAT": [{"synset": "27365", "text": "location", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8693705", "text": "urban area", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7991473", "text": "gathering", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8242502", "text": "municipality", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8643858", "text": "municipality", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-4.0", "type": "WordnetLAT"}, {"text": "city", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Nelson Mandela", "cookedLabel": "Nelson Mandela", "pageID": "21492751", "editDist": 0.0, "labelProbability": 0.996993, "logPopularity": 5.749392985908253, "score": 0.9760535385100926, "getByLAT": 0, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr003620", "qText": "what is currency in panama?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "4923519", "text": "property", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "13394134", "text": "medium of exchange", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7275291", "text": "standard", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33914", "text": "measure", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4772610", "text": "prevalence", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5051824", "text": "temporal arrangement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5051679", "text": "temporal property", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "4771667", "text": "generality", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "13598374", "text": "system of measurement", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5057266", "text": "presentness", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5053160", "text": "timing", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "currency", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Currency", "cookedLabel": "Currency", "pageID": "5665", "editDist": 0.0, "labelProbability": 0.800259, "logPopularity": 3.2188758248682006, "score": 0.38218402154685344, "getByLAT": 1, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Panama", "cookedLabel": "Panama", "pageID": "22997", "editDist": 0.0, "labelProbability": 0.697723, "logPopularity": 7.716906135298388, "score": 0.8985798300836622, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "currency in panama", "type": "ClueSubjectPhrase", "weight": 2.7}] }, +{"qId": "wqr003640", "qText": "what type of voting system does australia have?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "33319", "text": "communication", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "7957410", "text": "biological group", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "2855782", "text": "block", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847533", "text": "kind", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6830481", "text": "written symbol", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "6819327", "text": "symbol", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "6831828", "text": "character", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "6804229", "text": "signal", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "9628463", "text": "adult", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8008892", "text": "taxonomic group", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "type", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Voting system", "cookedLabel": "Voting system", "pageID": "29066482", "editDist": 0.0, "labelProbability": 0.955556, "logPopularity": 5.288267030694535, "score": 0.9041743837111924, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Australia", "cookedLabel": "Australia", "pageID": "4689264", "editDist": 0.0, "labelProbability": 0.747368, "logPopularity": 10.900768235614668, "score": 0.9951120776945014, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr003660", "qText": "who started southwest airlines?", "SV": ["started"], "lemmaSV": ["start"], "LAT": [{"synset": "7846", "text": "person", "specificity": "0.0", "type": "QuestionWordLAT"}], "Concept": [{"fullLabel": "Southwest Airlines", "cookedLabel": "Southwest Airlines", "pageID": "63032", "editDist": 0.0, "labelProbability": 0.998978, "logPopularity": 5.0369526024136295, "score": 0.9083202536141494, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "southwest airlines?", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003680", "qText": "who does new zealand import from?", "SV": ["import"], "lemmaSV": ["import"], "LAT": [{"synset": "2684", "text": "object", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "9339360", "text": "island", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "9357302", "text": "land", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "zealand", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "New Zealand", "cookedLabel": "New Zealand", "pageID": "4913064", "editDist": 0.0, "labelProbability": 0.681166, "logPopularity": 9.861310473636943, "score": 0.9641762949090023, "getByLAT": 1, "getByNE": 1, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [] }, +{"qId": "wqr003700", "qText": "what kind of political system is canada?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "5817200", "text": "content", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5847274", "text": "category", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5844071", "text": "concept", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "kind", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Coal Run Village, Kentucky", "cookedLabel": "Coal Run Village, Kentucky", "pageID": "115405", "editDist": 0.0, "labelProbability": 0.142857, "logPopularity": 4.718498871295094, "score": 0.05631447464088684, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Political system", "cookedLabel": "Political system", "pageID": "258724", "editDist": 0.0, "labelProbability": 0.238095, "logPopularity": 2.1972245773362196, "score": 0.05176398693827355, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Politics of Germany", "cookedLabel": "Politics of Germany", "pageID": "11935", "editDist": 0.0, "labelProbability": 0.190476, "logPopularity": 4.1588830833596715, "score": 0.050415590449092426, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}, {"fullLabel": "Canada", "cookedLabel": "Canada", "pageID": "5042916", "editDist": 0.0, "labelProbability": 0.755844, "logPopularity": 11.329182899020827, "score": 0.9963598807069324, "getByLAT": 0, "getByNE": 0, "getBySubject": 1, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "political system", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003720", "qText": "what countries are included in the netherlands?", "SV": ["included"], "lemmaSV": ["include"], "LAT": [{"synset": "27365", "text": "location", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "8648560", "text": "region", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8024893", "text": "organization", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "8508836", "text": "administrative district", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7958392", "text": "people", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8206589", "text": "unit", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8569713", "text": "district", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "8376876", "text": "political unit", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "8591861", "text": "geographical area", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7967506", "text": "social group", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "31563", "text": "group", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "countries", "specificity": "0.0", "type": "LAT"}, {"text": "country", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Netherlands", "cookedLabel": "Netherlands", "pageID": "21148", "editDist": 0.0, "labelProbability": 0.625727, "logPopularity": 10.223031598136654, "score": 0.9756182783081621, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}, {"fullLabel": "Netherlands national football team", "cookedLabel": "Netherlands national football team", "pageID": "9647657", "editDist": 0.0, "labelProbability": 0.0556568, "logPopularity": 7.611842399580417, "score": 0.18487179702788376, "getByLAT": 0, "getByNE": 1, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 0, "getByCWLookup": 1}], "Clue": [{"label": "netherlands", "type": "ClueNE", "weight": 1.11}] }, +{"qId": "wqr003740", "qText": "what are the major imports and exports of canada?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "9652940", "text": "traveler", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "10123254", "text": "foreigner", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "4731092", "text": "quality", "specificity": "-5.0", "type": "WordnetLAT"}, {"synset": "5145753", "text": "value", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "24444", "text": "attribute", "specificity": "-6.0", "type": "WordnetLAT"}, {"synset": "23451", "text": "cognition", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5842164", "text": "idea", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "3080712", "text": "commodity", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5145473", "text": "worth", "specificity": "-4.0", "type": "WordnetLAT"}, {"synset": "5177340", "text": "significance", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "7846", "text": "person", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6611268", "text": "message", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5928460", "text": "meaning", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "5817200", "text": "content", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "5175788", "text": "importance", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "33319", "text": "communication", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "imports", "specificity": "0.0", "type": "LAT"}, {"text": "import", "specificity": "0.0", "type": "ImplicitQLAT"}], "Concept": [{"fullLabel": "Major", "cookedLabel": "Major", "pageID": "201920", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 6.82001636467413, "score": 0.16955685053201597, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "The Major", "cookedLabel": "The Major", "pageID": "9600545", "editDist": 0.0, "labelProbability": 0.0, "logPopularity": 2.995732273553991, "score": 0.020167690730618193, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 0}, {"fullLabel": "Canada", "cookedLabel": "Canada", "pageID": "5042916", "editDist": 0.0, "labelProbability": 0.755844, "logPopularity": 11.329182899020827, "score": 0.9902048363115007, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "the major imports and exports of canada", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "major imports and exports of canada", "type": "CluePhrase", "weight": 0.99}, {"label": "major imports and exports", "type": "CluePhrase", "weight": 0.99}, {"label": "the major", "type": "ClueNE", "weight": 1.11}, {"label": "imports", "type": "ClueSubjectToken", "weight": 2.5}, {"label": "exports", "type": "ClueToken", "weight": 1.0}] }, +{"qId": "wqr003760", "qText": "what movie is angelina jolie directing?", "SV": [], "lemmaSV": [], "LAT": [{"synset": "4014270", "text": "product", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "22119", "text": "artifact", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "29677", "text": "event", "specificity": "-3.0", "type": "WordnetLAT"}, {"synset": "6631572", "text": "show", "specificity": "-1.0", "type": "WordnetLAT"}, {"synset": "3133774", "text": "creation", "specificity": "-2.0", "type": "WordnetLAT"}, {"synset": "7303344", "text": "social event", "specificity": "-2.0", "type": "WordnetLAT"}, {"text": "movie", "specificity": "0.0", "type": "LAT"}], "Concept": [{"fullLabel": "Angelina Jolie", "cookedLabel": "Angelina Jolie", "pageID": "5792809", "editDist": 0.0, "labelProbability": 0.995458, "logPopularity": 5.575949103146316, "score": 0.9059818840251335, "getByLAT": 0, "getByNE": 0, "getBySubject": 0, "getByNgram": 0, "getByFuzzyLookup": 1, "getByCWLookup": 1}], "Clue": [{"label": "angelina jolie directing", "type": "ClueSubjectPhrase", "weight": 2.7}, {"label": "directing", "type": "ClueSubjectToken", "weight": 2.5}] } +] diff --git a/evals/benchmark/stresscli/dataset/input_llama_3_1.jsonl b/evals/benchmark/stresscli/dataset/input_llama_3_1.jsonl new file mode 100644 index 00000000..b87c5f49 --- /dev/null +++ b/evals/benchmark/stresscli/dataset/input_llama_3_1.jsonl @@ -0,0 +1,10 @@ +{ + "5" : "Introduce Intel Corporation", + "25" : "In what ways does Intel Corporation enhance the computing performance, contributing to advancements in AI, data centers, and personal computing?", + "50" : "As a leading technology company, how does Intel Corporation innovate in semiconductor design and manufacturing, ensuring high performance and energy efficiency? What are the key areas of Intel's research and development that contribute to advancements in AI, data centers, and personal computing?", + "100" : "In the context of rapid technological evolution, how does Intel Corporation maintain its leadership in semiconductor innovation, addressing performance, energy efficiency, and scalability? How does Intel's research and development in AI, data centers, and personal computing contribute to the industry's growth? Furthermore, how does Intel ensure ethical considerations and sustainability in its production processes, while fostering inclusivity and diversity within the company? How does Intel's global reach and strategic partnerships enhance its ability to deliver cutting-edge technology solutions for industries and consumers?", + "500" : "In the context of the rapidly evolving technology landscape, how does Intel Corporation maintain its leadership in semiconductor innovation, addressing the complex challenges of performance, energy efficiency, and scalability in modern computing? How do Intel's research and development initiatives in AI, data centers, and personal computing drive technological advancements and support the growth of various sectors, including healthcare, automotive, and industrial applications? Additionally, how does Intel ensure ethical considerations, sustainability, and corporate responsibility are prioritized in its production processes, promoting environmentally friendly practices and responsible resource use? How does Intel foster a culture of inclusivity and diversity within its workforce, encouraging innovation, collaboration, and equal opportunities for all employees? Furthermore, how does Intel leverage its extensive global presence and strategic partnerships to deliver cutting-edge technology solutions, catering to the diverse needs of industries and consumers worldwide, while adhering to high standards of quality, reliability, and security? How does Intel's commitment to continuous improvement and customer satisfaction drive its efforts to create value and positive impact in the global technology ecosystem? how does Intel's approach to advanced manufacturing technologies, such as extreme ultraviolet (EUV) lithography and 3D packaging, contribute to the production of next-generation processors with higher performance and lower power consumption? How does Intel's investment in quantum computing and neuromorphic computing research aim to revolutionize future computing paradigms and solve problems that are currently intractable with classical computing methods? What specific strategies does Intel employ to mitigate supply chain risks and ensure a stable supply of critical materials and components amidst global uncertainties? Moreover, how does Intel's corporate social responsibility initiatives, including its efforts in reducing greenhouse gas emissions and transitioning to renewable energy sources, align with global sustainability goals and enhance its reputation as a responsible corporate citizen? How does Intel's educational outreach and workforce development programs help bridge the skills gap in the technology sector and prepare the next generation of innovators and engineers? In what ways does Intel engage with local communities and contribute to socioeconomic development through its various community engagement programs and philanthropic activities? Lastly, how does Intel's adherence to rigorous security standards and its proactive approach to cybersecurity ensure the protection of data and intellectual property, instilling trust among its customers and stakeholders? How does Intel's collaborative efforts with industry partners, academic institutions, and government agencies foster an ecosystem of innovation and drive forward the collective progress in science and technology? How does Intel continuously adapt its business strategies to stay ahead of market trends and maintain its competitive edge in the ever-changing global technology market?", + "1k" : "In the context of the rapidly evolving technology landscape, Intel Corporation maintains its leadership in semiconductor innovation by consistently addressing the complex challenges of performance, energy efficiency, and scalability in modern computing. It's achieved through substantial investment in research and development, as well as leveraging cutting-edge manufacturing technologies. Intel's initiatives in AI, data centers, and personal computing are pivotal in driving technological advancements that support the growth of various sectors, including healthcare, automotive, and industrial applications. Intel's research and development initiatives are designed to keep it at the forefront of innovation. In AI, Intel is developing processors and accelerators that optimize machine learning and deep learning workloads, making AI more accessible and efficient across different industries. In data centers, Intel's advancements in high-performance computing and networking enable faster data processing and storage solutions, which are crucial for handling the exponential growth of data. In personal computing, Intel continuously improves its CPU architectures to deliver higher performance and energy efficiency, catering to both consumer and enterprise needs. Ensuring ethical considerations, sustainability, and corporate responsibility are top priorities in Intel's production processes. The company promotes environmentally friendly practices and responsible resource use through initiatives like reducing greenhouse gas emissions, transitioning to renewable energy sources, and minimizing waste. Intel also emphasizes the ethical use of AI and technology, ensuring that innovations are developed and deployed responsibly, with considerations for their societal impacts. Intel fosters a culture of inclusivity and diversity within its workforce, recognizing that a diverse team drives innovation and collaboration. The company has implemented various programs to encourage equal opportunities for all employees, such as diversity hiring practices, mentorship programs, and employee resource groups. By creating an inclusive workplace, Intel ensures that all employees can contribute their unique perspectives and talents, fostering a collaborative and innovative environment. Leveraging its extensive global presence and strategic partnerships, Intel delivers cutting-edge technology solutions that cater to the diverse needs of industries and consumers worldwide. The company adheres to high standards of quality, reliability, and security, ensuring that its products meet the stringent demands of different markets. Intel's commitment to continuous improvement and customer satisfaction drives its efforts to create value and positive impact in the global technology ecosystem. Intel's approach to advanced manufacturing technologies, such as extreme ultraviolet (EUV) lithography and 3D packaging, plays a critical role in producing next-generation processors with higher performance and lower power consumption. These technologies enable Intel to push the boundaries of Moore's Law, delivering more powerful and efficient chips that power a wide range of applications. Investment in quantum computing and neuromorphic computing research is another area where Intel aims to revolutionize future computing paradigms. Quantum computing holds the potential to solve complex problems that are currently intractable with classical computing methods, while neuromorphic computing mimics the human brain's neural networks, leading to advancements in AI and machine learning. To mitigate supply chain risks and ensure a stable supply of critical materials and components amidst global uncertainties, Intel employs a multi-faceted strategy. This includes diversifying its supplier base, investing in local manufacturing capabilities, and closely monitoring the supply chain to identify and address potential disruptions proactively. Intel's corporate social responsibility (CSR) initiatives, including efforts to reduce greenhouse gas emissions and transition to renewable energy sources, align with global sustainability goals. These initiatives enhance Intel's reputation as a responsible corporate citizen and contribute to a more sustainable future. Intel's educational outreach and workforce development programs help bridge the skills gap in the technology sector, preparing the next generation of innovators and engineers. Through various community engagement programs and philanthropic activities, Intel engages with local communities and contributes to socioeconomic development. These efforts include supporting STEM education, promoting digital literacy, and providing resources for underserved communities, ensuring that technological advancements benefit society as a whole. Adherence to rigorous security standards and a proactive approach to cybersecurity are integral to Intel's operations, ensuring the protection of data and intellectual property. This instills trust among customers and stakeholders, reinforcing Intel's commitment to providing secure and reliable technology solutions. Collaborative efforts with industry partners, academic institutions, and government agencies foster an ecosystem of innovation, driving forward the collective progress in science and technology. Intel continuously adapts its business strategies to stay ahead of market trends, maintaining its competitive edge in the ever-changing global technology market. Intel's extensive patent portfolio plays a crucial role in securing its position as an industry leader and facilitating cross-industry collaborations. The company's commitment to diversity, equity, and inclusion manifests in its hiring practices, workplace culture, and external partnerships, ensuring that underrepresented groups are given ample opportunities to thrive. Initiatives focused on ethical AI development and responsible use of technology address the societal impacts of Intel's innovations, ensuring that advancements in AI and computing are used for the greater good. Intel's engagement in global policy discussions influences regulations and standards in the technology sector, contributing to a balanced and fair technological landscape. Overall, Intel's multifaceted approach to innovation, sustainability, and corporate responsibility positions it as a leader in the semiconductor industry.", + "2k" : "In the context of the rapidly evolving technology landscape, Intel Corporation maintains its leadership in semiconductor innovation by consistently addressing the complex challenges of performance, energy efficiency, and scalability in modern computing. It's achieved through substantial investment in research and development, as well as leveraging cutting-edge manufacturing technologies. Intel's initiatives in AI, data centers, and personal computing are pivotal in driving technological advancements that support the growth of various sectors, including healthcare, automotive, and industrial applications. Intel's research and development initiatives are designed to keep it at the forefront of innovation. In AI, Intel is developing processors and accelerators that optimize machine learning and deep learning workloads, making AI more accessible and efficient across different industries. In data centers, Intel's advancements in high-performance computing and networking enable faster data processing and storage solutions, which are crucial for handling the exponential growth of data. In personal computing, Intel continuously improves its CPU architectures to deliver higher performance and energy efficiency, catering to both consumer and enterprise needs. Ensuring ethical considerations, sustainability, and corporate responsibility are top priorities in Intel's production processes. The company promotes environmentally friendly practices and responsible resource use through initiatives like reducing greenhouse gas emissions, transitioning to renewable energy sources, and minimizing waste. Intel also emphasizes the ethical use of AI and technology, ensuring that innovations are developed and deployed responsibly, with considerations for their societal impacts. Intel fosters a culture of inclusivity and diversity within its workforce, recognizing that a diverse team drives innovation and collaboration. The company has implemented various programs to encourage equal opportunities for all employees, such as diversity hiring practices, mentorship programs, and employee resource groups. By creating an inclusive workplace, Intel ensures that all employees can contribute their unique perspectives and talents, fostering a collaborative and innovative environment. Leveraging its extensive global presence and strategic partnerships, Intel delivers cutting-edge technology solutions that cater to the diverse needs of industries and consumers worldwide. The company adheres to high standards of quality, reliability, and security, ensuring that its products meet the stringent demands of different markets. Intel's commitment to continuous improvement and customer satisfaction drives its efforts to create value and positive impact in the global technology ecosystem. Intel's approach to advanced manufacturing technologies, such as extreme ultraviolet (EUV) lithography and 3D packaging, plays a critical role in producing next-generation processors with higher performance and lower power consumption. These technologies enable Intel to push the boundaries of Moore's Law, delivering more powerful and efficient chips that power a wide range of applications. Investment in quantum computing and neuromorphic computing research is another area where Intel aims to revolutionize future computing paradigms. Quantum computing holds the potential to solve complex problems that are currently intractable with classical computing methods, while neuromorphic computing mimics the human brain's neural networks, leading to advancements in AI and machine learning. To mitigate supply chain risks and ensure a stable supply of critical materials and components amidst global uncertainties, Intel employs a multi-faceted strategy. This includes diversifying its supplier base, investing in local manufacturing capabilities, and closely monitoring the supply chain to identify and address potential disruptions proactively. Intel's corporate social responsibility (CSR) initiatives, including efforts to reduce greenhouse gas emissions and transition to renewable energy sources, align with global sustainability goals. These initiatives enhance Intel's reputation as a responsible corporate citizen and contribute to a more sustainable future. Intel's educational outreach and workforce development programs help bridge the skills gap in the technology sector, preparing the next generation of innovators and engineers. Through various community engagement programs and philanthropic activities, Intel engages with local communities and contributes to socioeconomic development. These efforts include supporting STEM education, promoting digital literacy, and providing resources for underserved communities, ensuring that technological advancements benefit society as a whole. Adherence to rigorous security standards and a proactive approach to cybersecurity are integral to Intel's operations, ensuring the protection of data and intellectual property. This instills trust among customers and stakeholders, reinforcing Intel's commitment to providing secure and reliable technology solutions. Collaborative efforts with industry partners, academic institutions, and government agencies foster an ecosystem of innovation, driving forward the collective progress in science and technology. Intel continuously adapts its business strategies to stay ahead of market trends, maintaining its competitive edge in the ever-changing global technology market. Intel's extensive patent portfolio plays a crucial role in securing its position as an industry leader and facilitating cross-industry collaborations. The company's commitment to diversity, equity, and inclusion manifests in its hiring practices, workplace culture, and external partnerships, ensuring that underrepresented groups are given ample opportunities to thrive. Initiatives focused on ethical AI development and responsible use of technology address the societal impacts of Intel's innovations, ensuring that advancements in AI and computing are used for the greater good. Intel's engagement in global policy discussions influences regulations and standards in the technology sector, contributing to a balanced and fair technological landscape. Overall, Intel's multifaceted approach to innovation, sustainability, and corporate responsibility positions it as a leader in the semiconductor industry. In addition to these initiatives, Intel is deeply involved in fostering the next generation of tech talent through its partnerships with educational institutions. The company supports numerous scholarship programs and research grants that enable students and researchers to explore new frontiers in technology. Intel’s investment in education not only nurtures future innovators but also ensures a continuous pipeline of skilled professionals who can contribute to the tech industry. Moreover, Intel’s focus on environmental sustainability extends beyond its manufacturing processes. The company is actively involved in developing technologies that reduce the environmental impact of its products. For instance, Intel is working on energy-efficient data centers that use less power and water, thereby reducing their carbon footprint. Intel’s efforts in creating sustainable technology solutions are aligned with global efforts to combat climate change and promote environmental stewardship. Intel's strategic vision also encompasses the development of smart cities and IoT (Internet of Things) ecosystems. By integrating advanced sensors, connectivity, and AI, Intel is enabling cities to become more efficient, resilient, and responsive to the needs of their inhabitants. These smart city solutions include intelligent traffic management systems, energy-efficient buildings, and enhanced public safety measures, all of which contribute to improved urban living conditions. Additionally, Intel is committed to ethical business practices and transparency in its operations. The company adheres to strict ethical guidelines and regularly publishes reports on its corporate governance, environmental impact, and social responsibility initiatives. This transparency not only builds trust with stakeholders but also sets a benchmark for other companies in the industry to follow. Intel's role in the development of 5G technology is another area where the company is making significant strides. By providing the necessary hardware and software solutions, Intel is helping to accelerate the deployment of 5G networks worldwide. This next-generation connectivity will enable faster and more reliable internet access, supporting innovations in areas such as autonomous vehicles, telemedicine, and remote education. Furthermore, Intel’s proactive approach to cybersecurity ensures that its products and services are safeguarded against emerging threats. The company invests heavily in developing advanced security technologies and collaborates with global cybersecurity experts to stay ahead of potential vulnerabilities. This commitment to security not only protects Intel’s assets but also provides peace of mind to its customers and partners. Intel's dedication to fostering a culture of continuous improvement is evident in its internal processes and external collaborations. The company regularly reviews its strategies and operations to identify areas for enhancement and innovation. By embracing a growth mindset, Intel ensures that it remains agile and responsive to the dynamic needs of the tech industry. Finally, Intel's influence extends beyond the tech sector through its active participation in policy advocacy and industry standards development. By engaging with policymakers, industry leaders, and international organizations, Intel helps shape the regulatory landscape to support innovation and ensure fair competition. This advocacy work is crucial in creating an environment where technological advancements can thrive and benefit society as a whole. Through its comprehensive approach to innovation, sustainability, and corporate responsibility, Intel continues to lead the semiconductor industry into the future, driving progress and creating value for its customers, partners, and the global community. In addition to these initiatives, Intel is leveraging artificial intelligence to enhance its supply chain operations, ensuring that materials and products are delivered efficiently and sustainably. By utilizing AI-driven analytics and predictive modeling, Intel can anticipate supply chain disruptions and optimize logistics, reducing both costs and environmental impact. Intel's commitment to open-source development is another key aspect of its innovation strategy. The company actively contributes to and supports a variety of open-source projects, fostering collaboration and accelerating technological advancements across the industry. This not only helps in the rapid development of new technologies but also ensures that these innovations are accessible to a broader audience, promoting inclusivity in tech. Furthermore, Intel's advancements in edge computing are transforming industries by enabling real-time data processing at the source of data generation. This is particularly important for applications in healthcare, manufacturing, and autonomous systems, where low latency and high reliability are critical. By bringing computing power closer to where data is generated, Intel is helping to unlock new possibilities for innovation and efficiency. Intel's focus on developing partnerships with other technology leaders, academic institutions, and government agencies amplifies its impact. Collaborative projects in areas such as AI ethics, cybersecurity, and next-generation computing not only drive technological progress but also address broader societal challenges. These partnerships are crucial in creating a holistic approach to innovation, ensuring that technological advancements are beneficial and sustainable. Moreover, Intel's investment in renewable energy projects demonstrates its commitment to environmental stewardship. The company is involved in various initiatives to increase the use of solar, wind, and other renewable energy sources in its operations. This not only reduces Intel's carbon footprint but also supports the global transition to sustainable energy solutions. Intel's efforts in promoting digital inclusion are also noteworthy. The company supports programs that provide digital skills training and access to technology for underserved communities. By bridging the digital divide, Intel is empowering individuals and communities to participate in the digital economy, fostering economic growth and social inclusion. Intel continues to shape the future of technology.", + "3k" : "In the context of the rapidly evolving technology landscape, Intel Corporation maintains its leadership in semiconductor innovation by consistently addressing the complex challenges of performance, energy efficiency, and scalability in modern computing. It's achieved through substantial investment in research and development, as well as leveraging cutting-edge manufacturing technologies. Intel's initiatives in AI, data centers, and personal computing are pivotal in driving technological advancements that support the growth of various sectors, including healthcare, automotive, and industrial applications. Intel's research and development initiatives are designed to keep it at the forefront of innovation. In AI, Intel is developing processors and accelerators that optimize machine learning and deep learning workloads, making AI more accessible and efficient across different industries. In data centers, Intel's advancements in high-performance computing and networking enable faster data processing and storage solutions, which are crucial for handling the exponential growth of data. In personal computing, Intel continuously improves its CPU architectures to deliver higher performance and energy efficiency, catering to both consumer and enterprise needs. Ensuring ethical considerations, sustainability, and corporate responsibility are top priorities in Intel's production processes. The company promotes environmentally friendly practices and responsible resource use through initiatives like reducing greenhouse gas emissions, transitioning to renewable energy sources, and minimizing waste. Intel also emphasizes the ethical use of AI and technology, ensuring that innovations are developed and deployed responsibly, with considerations for their societal impacts. Intel fosters a culture of inclusivity and diversity within its workforce, recognizing that a diverse team drives innovation and collaboration. The company has implemented various programs to encourage equal opportunities for all employees, such as diversity hiring practices, mentorship programs, and employee resource groups. By creating an inclusive workplace, Intel ensures that all employees can contribute their unique perspectives and talents, fostering a collaborative and innovative environment. Leveraging its extensive global presence and strategic partnerships, Intel delivers cutting-edge technology solutions that cater to the diverse needs of industries and consumers worldwide. The company adheres to high standards of quality, reliability, and security, ensuring that its products meet the stringent demands of different markets. Intel's commitment to continuous improvement and customer satisfaction drives its efforts to create value and positive impact in the global technology ecosystem. Intel's approach to advanced manufacturing technologies, such as extreme ultraviolet (EUV) lithography and 3D packaging, plays a critical role in producing next-generation processors with higher performance and lower power consumption. These technologies enable Intel to push the boundaries of Moore's Law, delivering more powerful and efficient chips that power a wide range of applications. Investment in quantum computing and neuromorphic computing research is another area where Intel aims to revolutionize future computing paradigms. Quantum computing holds the potential to solve complex problems that are currently intractable with classical computing methods, while neuromorphic computing mimics the human brain's neural networks, leading to advancements in AI and machine learning. To mitigate supply chain risks and ensure a stable supply of critical materials and components amidst global uncertainties, Intel employs a multi-faceted strategy. This includes diversifying its supplier base, investing in local manufacturing capabilities, and closely monitoring the supply chain to identify and address potential disruptions proactively. Intel's corporate social responsibility (CSR) initiatives, including efforts to reduce greenhouse gas emissions and transition to renewable energy sources, align with global sustainability goals. These initiatives enhance Intel's reputation as a responsible corporate citizen and contribute to a more sustainable future. Intel's educational outreach and workforce development programs help bridge the skills gap in the technology sector, preparing the next generation of innovators and engineers. Through various community engagement programs and philanthropic activities, Intel engages with local communities and contributes to socioeconomic development. These efforts include supporting STEM education, promoting digital literacy, and providing resources for underserved communities, ensuring that technological advancements benefit society as a whole. Adherence to rigorous security standards and a proactive approach to cybersecurity are integral to Intel's operations, ensuring the protection of data and intellectual property. This instills trust among customers and stakeholders, reinforcing Intel's commitment to providing secure and reliable technology solutions. Collaborative efforts with industry partners, academic institutions, and government agencies foster an ecosystem of innovation, driving forward the collective progress in science and technology. Intel continuously adapts its business strategies to stay ahead of market trends, maintaining its competitive edge in the ever-changing global technology market. Intel's extensive patent portfolio plays a crucial role in securing its position as an industry leader and facilitating cross-industry collaborations. The company's commitment to diversity, equity, and inclusion manifests in its hiring practices, workplace culture, and external partnerships, ensuring that underrepresented groups are given ample opportunities to thrive. Initiatives focused on ethical AI development and responsible use of technology address the societal impacts of Intel's innovations, ensuring that advancements in AI and computing are used for the greater good. Intel's engagement in global policy discussions influences regulations and standards in the technology sector, contributing to a balanced and fair technological landscape. Overall, Intel's multifaceted approach to innovation, sustainability, and corporate responsibility positions it as a leader in the semiconductor industry. In addition to these initiatives, Intel is deeply involved in fostering the next generation of tech talent through its partnerships with educational institutions. The company supports numerous scholarship programs and research grants that enable students and researchers to explore new frontiers in technology. Intel’s investment in education not only nurtures future innovators but also ensures a continuous pipeline of skilled professionals who can contribute to the tech industry. Moreover, Intel’s focus on environmental sustainability extends beyond its manufacturing processes. The company is actively involved in developing technologies that reduce the environmental impact of its products. For instance, Intel is working on energy-efficient data centers that use less power and water, thereby reducing their carbon footprint. Intel’s efforts in creating sustainable technology solutions are aligned with global efforts to combat climate change and promote environmental stewardship. Intel's strategic vision also encompasses the development of smart cities and IoT (Internet of Things) ecosystems. By integrating advanced sensors, connectivity, and AI, Intel is enabling cities to become more efficient, resilient, and responsive to the needs of their inhabitants. These smart city solutions include intelligent traffic management systems, energy-efficient buildings, and enhanced public safety measures, all of which contribute to improved urban living conditions. Additionally, Intel is committed to ethical business practices and transparency in its operations. The company adheres to strict ethical guidelines and regularly publishes reports on its corporate governance, environmental impact, and social responsibility initiatives. This transparency not only builds trust with stakeholders but also sets a benchmark for other companies in the industry to follow. Intel's role in the development of 5G technology is another area where the company is making significant strides. By providing the necessary hardware and software solutions, Intel is helping to accelerate the deployment of 5G networks worldwide. This next-generation connectivity will enable faster and more reliable internet access, supporting innovations in areas such as autonomous vehicles, telemedicine, and remote education. Furthermore, Intel’s proactive approach to cybersecurity ensures that its products and services are safeguarded against emerging threats. The company invests heavily in developing advanced security technologies and collaborates with global cybersecurity experts to stay ahead of potential vulnerabilities. This commitment to security not only protects Intel’s assets but also provides peace of mind to its customers and partners. Intel's dedication to fostering a culture of continuous improvement is evident in its internal processes and external collaborations. The company regularly reviews its strategies and operations to identify areas for enhancement and innovation. By embracing a growth mindset, Intel ensures that it remains agile and responsive to the dynamic needs of the tech industry. Finally, Intel's influence extends beyond the tech sector through its active participation in policy advocacy and industry standards development. By engaging with policymakers, industry leaders, and international organizations, Intel helps shape the regulatory landscape to support innovation and ensure fair competition. This advocacy work is crucial in creating an environment where technological advancements can thrive and benefit society as a whole. Through its comprehensive approach to innovation, sustainability, and corporate responsibility, Intel continues to lead the semiconductor industry into the future, driving progress and creating value for its customers, partners, and the global community. In addition to these initiatives, Intel is leveraging artificial intelligence to enhance its supply chain operations, ensuring that materials and products are delivered efficiently and sustainably. By utilizing AI-driven analytics and predictive modeling, Intel can anticipate supply chain disruptions and optimize logistics, reducing both costs and environmental impact. Intel's commitment to open-source development is another key aspect of its innovation strategy. The company actively contributes to and supports a variety of open-source projects, fostering collaboration and accelerating technological advancements across the industry. This not only helps in the rapid development of new technologies but also ensures that these innovations are accessible to a broader audience, promoting inclusivity in tech. Furthermore, Intel's advancements in edge computing are transforming industries by enabling real-time data processing at the source of data generation. This is particularly important for applications in healthcare, manufacturing, and autonomous systems, where low latency and high reliability are critical. By bringing computing power closer to where data is generated, Intel is helping to unlock new possibilities for innovation and efficiency. Intel's focus on developing partnerships with other technology leaders, academic institutions, and government agencies amplifies its impact. Collaborative projects in areas such as AI ethics, cybersecurity, and next-generation computing not only drive technological progress but also address broader societal challenges. These partnerships are crucial in creating a holistic approach to innovation, ensuring that technological advancements are beneficial and sustainable. Moreover, Intel's investment in renewable energy projects demonstrates its commitment to environmental stewardship. The company is involved in various initiatives to increase the use of solar, wind, and other renewable energy sources in its operations. This not only reduces Intel's carbon footprint but also supports the global transition to sustainable energy solutions. Intel's efforts in promoting digital inclusion are also noteworthy. The company supports programs that provide digital skills training and access to technology for underserved communities. By bridging the digital divide, Intel is empowering individuals and communities to participate in the digital economy, fostering economic growth and social inclusion. Intel continues to shape the future of technology. Intel’s commitment to advancing education extends beyond traditional boundaries. The company is actively involved in creating and supporting initiatives that promote STEM (Science, Technology, Engineering, and Mathematics) education worldwide. By partnering with educational institutions, Intel develops curricula that prepare students for careers in technology, ensuring that the future workforce is equipped with the skills needed to drive innovation. In addition, Intel’s push for smart manufacturing is revolutionizing the production process. By integrating AI, IoT, and advanced robotics into manufacturing, Intel enhances efficiency, reduces waste, and improves product quality. These smart manufacturing techniques are not only beneficial for Intel’s production capabilities but also set new standards for the industry, demonstrating how technology can transform traditional manufacturing. Intel’s efforts in digital health go beyond developing hardware solutions. The company collaborates with healthcare providers to implement AI-driven diagnostics and patient monitoring systems that provide real-time health data, enabling proactive medical care. These technologies are particularly crucial in managing chronic diseases and supporting remote healthcare, which has become increasingly important in the wake of global health challenges. The company is also making strides in the field of environmental monitoring and conservation. By deploying sensor networks and data analytics, Intel contributes to initiatives that track environmental changes, monitor wildlife, and manage natural resources more effectively. These efforts highlight Intel’s commitment to leveraging technology for the betterment of the planet. Intel’s active role in shaping digital policy and regulation underscores its influence in the tech industry. The company engages with policymakers and regulatory bodies to advocate for frameworks that support innovation while protecting consumer rights and promoting fair competition. This engagement ensures that the regulatory environment keeps pace with technological advancements, fostering an ecosystem conducive to growth and innovation. Moreover, Intel’s focus on ethical AI is setting benchmarks for the responsible development and deployment of artificial intelligence. The company collaborates with industry partners, academics, and ethicists to create guidelines that ensure AI technologies are developed with fairness, transparency, and accountability. This proactive approach aims to mitigate biases and ensure that AI serves the public good. In the realm of renewable energy, Intel is pioneering projects that integrate solar and wind power into its operations. The company is also exploring innovative energy storage solutions that enhance the reliability and efficiency of renewable energy sources. By committing to renewable energy, Intel not only reduces its carbon footprint but also demonstrates leadership in the transition to a sustainable energy future. Intel’s dedication to creating an inclusive digital world is reflected in its accessibility initiatives. The company designs products that cater to individuals with disabilities, ensuring that technology is usable and beneficial for everyone. This includes developing assistive technologies and ensuring that software and hardware are compatible with accessibility standards. In the field of financial services, Intel’s technologies are transforming the landscape by enabling faster and more secure transactions. The company’s innovations in blockchain, AI, and cloud computing support financial institutions in enhancing security, reducing fraud, and improving customer experiences. These advancements are critical in building trust and efficiency in the financial sector. Additionally, Intel’s vision for the future includes significant advancements in computational biology. By leveraging HPC and AI, Intel is enabling researchers to simulate biological processes at unprecedented scales, leading to breakthroughs in understanding diseases, developing new treatments, and advancing personalized medicine. Intel's investment in 3D printing and additive manufacturing is opening new possibilities in design and production. These technologies allow for the creation of complex structures that are not feasible with traditional manufacturing methods, reducing material waste and enabling rapid prototyping and production. The company's focus on edge AI is transforming industries by enabling real-time analytics and decision-making at the edge of networks. This is particularly important for applications in smart cities, industrial automation, and autonomous systems, where immediate processing and response are critical. Intel's strategic acquisitions and partnerships play a crucial role in its innovation ecosystem. By acquiring companies with complementary technologies and expertise, Intel expands its capabilities and accelerates the development of new solutions. These strategic moves ensure that Intel remains at the cutting edge of technological advancements. Finally, Intel’s ongoing commitment to corporate governance and ethical business practices ensures that it operates with integrity and accountability. The company’s governance frameworks are designed to uphold the highest standards of ethical conduct, transparency, and stakeholder engagement, reinforcing Intel’s reputation as a trusted leader in the tech industry. Through its comprehensive approach to innovation, sustainability, and corporate responsibility, Intel continues to lead the semiconductor industry into the future, driving progress and creating value for its customers, partners, and the global community. The company's multifaceted initiatives and strategic vision position it to address the challenges and opportunities of the digital age, ensuring that it remains a driving force in the global technology landscape. Intel's relentless pursuit of innovation, commitment to sustainability, and dedication to ethical practices solidify its leadership in the semiconductor industry. By advancing AI, fostering inclusivity, enhancing security, and driving environmental stewardship, Intel not only shapes the future of technology but also ensures its positive impact on society. Through strategic partnerships, cutting-edge research and proactive policy engagement, Intel continues to drive progress in a rapidly evolving digital world." +} diff --git a/evals/benchmark/stresscli/locust/README.md b/evals/benchmark/stresscli/locust/README.md new file mode 100644 index 00000000..da2b1f21 --- /dev/null +++ b/evals/benchmark/stresscli/locust/README.md @@ -0,0 +1,73 @@ +# locust scripts for OPEA ChatQnA + +Locust is an open source performance/load testing tool for HTTP and other protocols. Its developer-friendly approach lets you define your tests in regular Python code. + +Locust tests can be run from command line or using its web-based UI. Throughput, response times and errors can be viewed in real time and/or exported for later analysis. + +You can import regular Python libraries into your tests, and with Locust's pluggable architecture it is infinitely expandable. Unlike when using most other tools, your test design will never be limited by a GUI or domain-specific language. + +To get started right away, head over to the [documentation](http://docs.locust.io/en/stable/installation.html). + +## Configuration file + +locust.conf for configuration, need to modify this file to meet your requirement +``` +#Locust script file +locustfile = locustfile.py +#Run without html UI +headless = true +#Target address and port +host = http://10.233.23.72:8888 +#Conncurrency numbers +users = 16 +#Advice same with users, means no request shape required, and spawn to max at beginning +spawn-rate = 16 +#Set to a longer time, since max-request parameter will stop the benchmark +run-time = 100m +#After this number of request issued, benchmark will stopped +max-request = 16 +#Only log summary info +only-summary = true +``` + +## Basic Usage + +``` + pip install locust numpy + locust +``` + + This runs a benchmark using parameters defined in locust.conf + + Output: +``` +locust is working ... + +Concurrency : 4 +Max request count : 8 + +Exceed the max-request number:8 , Exit... +=================Total statistics===================== +Succeed Response: 8 (Total 8, 100.0% Success), Duration: 81.14s, Tokens: 1024, RPS: 0.10, Tokens per Second: 12.62 +End to End latency(ms), P50: 39860.39, P99: 45768.89 +First token latency(ms), P50: 6839.28, P99: 11074.14 +Average Next token latency(ms): 255.71 +Average token latency(ms) : 308.90 +====================================================== + + +[2024-07-11 08:36:37,795] node1/INFO/locust.main: Shutting down (exit code 0) +Type Name # reqs # fails | Avg Min Max Med | req/s failures/s +--------|--------------------------------|-------|-------------|-------|-------|-------|-------|--------|----------- +POST /v1/chatqna 8 0(0.00%) | 39539 32856 45833 36000 | 0.10 0.00 +--------|--------------------------------|-------|-------------|-------|-------|-------|-------|--------|----------- + Aggregated 8 0(0.00%) | 39539 32856 45833 36000 | 0.10 0.00 + +Response time percentiles (approximated) +Type Name 50% 66% 75% 80% 90% 95% 98% 99% 99.9% 99.99% 100% # reqs +--------|----------------------|--------|------|------|------|------|------|------|------|------|------|------|------ +POST /v1/chatqna 44000 45000 45000 45000 46000 46000 46000 46000 46000 46000 46000 8 +--------|----------------------|--------|------|------|------|------|------|------|------|------|------|------|------ + Aggregated 44000 45000 45000 45000 46000 46000 46000 46000 46000 46000 46000 8 + +``` diff --git a/evals/benchmark/stresscli/locust/__init__.py b/evals/benchmark/stresscli/locust/__init__.py new file mode 100644 index 00000000..916f3a44 --- /dev/null +++ b/evals/benchmark/stresscli/locust/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/evals/benchmark/stresscli/locust/aistress.py b/evals/benchmark/stresscli/locust/aistress.py new file mode 100644 index 00000000..86a1ecdf --- /dev/null +++ b/evals/benchmark/stresscli/locust/aistress.py @@ -0,0 +1,157 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging +import os +import sys +import threading +import time + +import gevent +import numpy +from locust import HttpUser, between, events, task +from locust.runners import STATE_CLEANUP, STATE_STOPPED, STATE_STOPPING, MasterRunner, WorkerRunner + +cwd = os.path.dirname(__file__) +sys.path.append(cwd) + + +@events.init_command_line_parser.add_listener +def _(parser): + parser.add_argument( + "--max-request", + type=int, + env_var="MAX_REQUEST", + default=10000, + help="Stop the benchmark If exceed this request", + ) + parser.add_argument( + "--http-timeout", type=int, env_var="HTTP_TIMEOUT", default=3000, help="Http timeout before receive response" + ) + parser.add_argument( + "--bench-target", + type=str, + env_var="BENCH_TARGET", + default="chatqnafixed", + help="python package name for benchmark target", + ) + + +reqlist = [] +start_ts = 0 +end_ts = 0 +req_total = 0 +last_resp_ts = 0 + +bench_package = "" +console_logger = logging.getLogger("locust.stats_logger") + + +class AiStressUser(HttpUser): + request = 0 + _lock = threading.Lock() + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @task + def bench_main(self): + + if AiStressUser.request >= self.environment.parsed_options.max_request: + time.sleep(1) + return + with AiStressUser._lock: + AiStressUser.request += 1 + self.environment.runner.send_message("worker_reqsent", 1) + tokens = 0 + start_ts = time.time() + # With stream=False here, Cannot get first response time,Workaround as response.elapsed.total_seconds() + url = bench_package.getUrl() + reqData = bench_package.getReqData() + try: + with self.client.post( + url, + json=reqData, + stream=False, + catch_response=True, + timeout=self.environment.parsed_options.http_timeout, + ) as resp: + logging.debug("Got response...........................") + first_resp = time.perf_counter() + + if resp.status_code >= 200 and resp.status_code < 400: + reqdata = bench_package.respStatics(self.environment, resp) + # reqlist.append(reqdata) + logging.debug(f"Request data collected {reqdata}") + self.environment.runner.send_message("worker_reqdata", reqdata) + logging.debug("Finished response analysis...........................") + except Exception as e: + # In case of exception occurs, locust lost the statistic for this request. + # Consider as a failed request, and report to Locust statistics + logging.error(f"Failed with request : {e}") + self.environment.runner.stats.log_request("POST", "/v1/chatqna", time.time() - start_ts, 0) + self.environment.runner.stats.log_error("POST", "/v1/chatqna", "Locust Request error") + + # def on_stop(self) -> None: + + +@events.test_start.add_listener +def on_test_start(environment, **kwargs): + if not isinstance(environment.runner, WorkerRunner): + console_logger.info(f"Concurrency : {environment.parsed_options.num_users}") + console_logger.info(f"Max request count : {environment.parsed_options.max_request}") + console_logger.info(f"Http timeout : {environment.parsed_options.http_timeout}\n") + console_logger.info(f"Benchmark target : {environment.parsed_options.bench_target}\n") + + +@events.init.add_listener +def on_locust_init(environment, **_kwargs): + global bench_package + try: + bench_package = __import__(environment.parsed_options.bench_target) + except ImportError: + return None + if not isinstance(environment.runner, WorkerRunner): + gevent.spawn(checker, environment) + environment.runner.register_message("worker_reqdata", on_reqdata) + environment.runner.register_message("worker_reqsent", on_reqsent) + if not isinstance(environment.runner, MasterRunner): + environment.runner.register_message("all_reqcnt", on_reqcount) + + +@events.quitting.add_listener +def on_locust_quitting(environment, **kwargs): + if isinstance(environment.runner, WorkerRunner): + logging.debug("Running in WorkerRunner, DO NOT print statistics") + return + + logging.debug("#####Running in MasterRunner, DO print statistics") + bench_package.staticsOutput(environment, reqlist) + + +def on_reqdata(msg, **kwargs): + logging.debug(f"Request data: {msg.data}") + reqlist.append(msg.data) + + +def on_reqsent(environment, msg, **kwargs): + logging.debug(f"request sent: {msg.data}") + global req_total + req_total += 1 + environment.runner.send_message("all_reqcnt", req_total) + + +def on_reqcount(msg, **kwargs): + logging.debug(f"Update total request: {msg.data}") + AiStressUser.request = msg.data + + +def checker(environment): + while environment.runner.state not in [STATE_STOPPING, STATE_STOPPED, STATE_CLEANUP]: + time.sleep(1) + if environment.runner.stats.num_requests >= environment.parsed_options.max_request: + logging.info(f"Exceed the max-request number:{environment.runner.stats.num_requests}, Exit...") + # while environment.runner.user_count > 0: + time.sleep(5) + environment.runner.quit() + return diff --git a/evals/benchmark/stresscli/locust/audioqnabench.py b/evals/benchmark/stresscli/locust/audioqnabench.py new file mode 100644 index 00000000..abf23458 --- /dev/null +++ b/evals/benchmark/stresscli/locust/audioqnabench.py @@ -0,0 +1,38 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import random + +import tokenresponse as token + +cwd = os.path.dirname(__file__) +filename = f"{cwd}/../dataset/chatqna.json" +qlist = [] +try: + with open(filename) as qfile: + qlist = json.load(qfile) +except: + logging.error(f"Question File open failed: {filename}") + exit() + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + qid = random.randint(1, 189) + logging.debug(f"Selected question: {qlist[qid]['qText']}") + + return {"messages": qlist[qid]["qText"], "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/audioqnafixed.py b/evals/benchmark/stresscli/locust/audioqnafixed.py new file mode 100644 index 00000000..9ed29e18 --- /dev/null +++ b/evals/benchmark/stresscli/locust/audioqnafixed.py @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import tokenresponse as token + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + return {"messages": "What is the revenue of Nike in last 10 years before 2023? Give me detail", "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/chatqnabench.py b/evals/benchmark/stresscli/locust/chatqnabench.py new file mode 100644 index 00000000..abf23458 --- /dev/null +++ b/evals/benchmark/stresscli/locust/chatqnabench.py @@ -0,0 +1,38 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import random + +import tokenresponse as token + +cwd = os.path.dirname(__file__) +filename = f"{cwd}/../dataset/chatqna.json" +qlist = [] +try: + with open(filename) as qfile: + qlist = json.load(qfile) +except: + logging.error(f"Question File open failed: {filename}") + exit() + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + qid = random.randint(1, 189) + logging.debug(f"Selected question: {qlist[qid]['qText']}") + + return {"messages": qlist[qid]["qText"], "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/chatqnafixed.py b/evals/benchmark/stresscli/locust/chatqnafixed.py new file mode 100644 index 00000000..9ed29e18 --- /dev/null +++ b/evals/benchmark/stresscli/locust/chatqnafixed.py @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import tokenresponse as token + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + return {"messages": "What is the revenue of Nike in last 10 years before 2023? Give me detail", "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/codegenbench.py b/evals/benchmark/stresscli/locust/codegenbench.py new file mode 100644 index 00000000..abf23458 --- /dev/null +++ b/evals/benchmark/stresscli/locust/codegenbench.py @@ -0,0 +1,38 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import random + +import tokenresponse as token + +cwd = os.path.dirname(__file__) +filename = f"{cwd}/../dataset/chatqna.json" +qlist = [] +try: + with open(filename) as qfile: + qlist = json.load(qfile) +except: + logging.error(f"Question File open failed: {filename}") + exit() + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + qid = random.randint(1, 189) + logging.debug(f"Selected question: {qlist[qid]['qText']}") + + return {"messages": qlist[qid]["qText"], "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/codegenfixed.py b/evals/benchmark/stresscli/locust/codegenfixed.py new file mode 100644 index 00000000..9ed29e18 --- /dev/null +++ b/evals/benchmark/stresscli/locust/codegenfixed.py @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import tokenresponse as token + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + return {"messages": "What is the revenue of Nike in last 10 years before 2023? Give me detail", "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/codetransbench.py b/evals/benchmark/stresscli/locust/codetransbench.py new file mode 100644 index 00000000..abf23458 --- /dev/null +++ b/evals/benchmark/stresscli/locust/codetransbench.py @@ -0,0 +1,38 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import random + +import tokenresponse as token + +cwd = os.path.dirname(__file__) +filename = f"{cwd}/../dataset/chatqna.json" +qlist = [] +try: + with open(filename) as qfile: + qlist = json.load(qfile) +except: + logging.error(f"Question File open failed: {filename}") + exit() + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + qid = random.randint(1, 189) + logging.debug(f"Selected question: {qlist[qid]['qText']}") + + return {"messages": qlist[qid]["qText"], "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/codetransfixed.py b/evals/benchmark/stresscli/locust/codetransfixed.py new file mode 100644 index 00000000..9ed29e18 --- /dev/null +++ b/evals/benchmark/stresscli/locust/codetransfixed.py @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import tokenresponse as token + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + return {"messages": "What is the revenue of Nike in last 10 years before 2023? Give me detail", "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/faqgenbench.py b/evals/benchmark/stresscli/locust/faqgenbench.py new file mode 100644 index 00000000..abf23458 --- /dev/null +++ b/evals/benchmark/stresscli/locust/faqgenbench.py @@ -0,0 +1,38 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import random + +import tokenresponse as token + +cwd = os.path.dirname(__file__) +filename = f"{cwd}/../dataset/chatqna.json" +qlist = [] +try: + with open(filename) as qfile: + qlist = json.load(qfile) +except: + logging.error(f"Question File open failed: {filename}") + exit() + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + qid = random.randint(1, 189) + logging.debug(f"Selected question: {qlist[qid]['qText']}") + + return {"messages": qlist[qid]["qText"], "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/faqgenfixed.py b/evals/benchmark/stresscli/locust/faqgenfixed.py new file mode 100644 index 00000000..9ed29e18 --- /dev/null +++ b/evals/benchmark/stresscli/locust/faqgenfixed.py @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import tokenresponse as token + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + return {"messages": "What is the revenue of Nike in last 10 years before 2023? Give me detail", "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/locust.conf b/evals/benchmark/stresscli/locust/locust.conf new file mode 100644 index 00000000..be6979b3 --- /dev/null +++ b/evals/benchmark/stresscli/locust/locust.conf @@ -0,0 +1,9 @@ +locustfile = aistress.py +headless = true +host = http://$(service_ip):$(service_port) +users = 16 +spawn-rate = 16 +max-request = 16 +processes = 4 +only-summary = true +loglevel=WARNING diff --git a/evals/benchmark/stresscli/locust/tgifixed.py b/evals/benchmark/stresscli/locust/tgifixed.py new file mode 100644 index 00000000..ef80d8f5 --- /dev/null +++ b/evals/benchmark/stresscli/locust/tgifixed.py @@ -0,0 +1,66 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging + +import numpy +import tokenresponse as token + +console_logger = logging.getLogger("locust.stats_logger") + + +def getUrl(): + return "/generate" + + +def getReqData(): + return {"inputs": "What is Machine Learning?", "parameters": {"max_new_tokens": 17, "do_sample": True}} + + +def respStatics(environment, resp): + global metrics_list + global tgi_metrics + + tgi_metrics = { + "total_time": int(resp.headers["x-total-time"]), + "queue_time": int(resp.headers["x-queue-time"]), + "gtokens": int(resp.headers["x-generated-tokens"]), + "time_per_token": int(resp.headers["x-time-per-token"]), + } + general_metrics = token.respStatics(environment, resp) + tgi_metrics.update(general_metrics) + return tgi_metrics + + +# return "" + + +def staticsOutput(environment, reqlist): + gtokens = [] + time_per_token = [] + queue_time = [] + + token.staticsOutput(environment, reqlist) + + if len(reqlist) == 0: + logging.debug(f"len(reqlist): {len(reqlist)}, skip printing") + return + for req in iter(reqlist): + gtokens.append(req["gtokens"]) + time_per_token.append(req["time_per_token"]) + queue_time.append(req["queue_time"]) + qtime_msg = "Queue time(ms), P50: {:.2f}, P99: {:.2f}, Avg: {:.2f}" + tptk_msg = "Average token latency(ms), P50: {:.2f}, P99: {:.2f}, Avg: {:.2f}" + gtk_msg = "Generated Token count, Avg: {:.2f}" + console_logger.warning("\n=================TGI statistics=====================") + console_logger.warning( + qtime_msg.format(numpy.percentile(queue_time, 50), numpy.percentile(queue_time, 99), numpy.average(queue_time)) + ) + console_logger.warning( + tptk_msg.format( + numpy.percentile(time_per_token, 50), numpy.percentile(time_per_token, 99), numpy.average(time_per_token) + ) + ) + console_logger.warning(gtk_msg.format(numpy.average(gtokens))) + console_logger.warning("======================================================\n\n") + logging.shutdown() diff --git a/evals/benchmark/stresscli/locust/tokenresponse.py b/evals/benchmark/stresscli/locust/tokenresponse.py new file mode 100644 index 00000000..8b7a6e8f --- /dev/null +++ b/evals/benchmark/stresscli/locust/tokenresponse.py @@ -0,0 +1,99 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging + +import numpy + +console_logger = logging.getLogger("locust.stats_logger") + + +def testFunc(): + print("TestFunc from token_response") + + +def respStatics(environment, resp): + #######About the statistic data calculation###### + # Note: Locust statistic data include all requests, failed request may impact the correctness + # Here we only statistic for success response(Http 2xx/3xx) + # + # Token count : measured by analysis the response content + # First token latency : measured by response.elapsed, "measures the time taken between sending + # the first byte of the request and finishing parsing the headers." + # https://requests.readthedocs.io/en/latest/api/#requests.Response.elapsed + # Next token latency : request_meta['response_time'], reuse and align with locust extension + # https://github.com/locustio/locust/blob/master/locust/clients.py#L193 + return { + "tokens": resp.text.count("data: b"), + "first_token": resp.elapsed.total_seconds() * 1000, + "next_token": resp.request_meta["response_time"] - resp.elapsed.total_seconds() * 1000, + } + + +def staticsOutput(environment, reqlist): + first_token = [] + next_token = [] + avg_token = [] + e2e_lat = [] + tokens = 0 + duration = environment.runner.stats.last_request_timestamp - environment.runner.stats.start_time + + if len(reqlist) == 0: + logging.debug(f"len(reqlist): {len(reqlist)}, skip printing") + return + for req in iter(reqlist): + first_token.append(req["first_token"]) + if req["tokens"] != 0: + next_token.append(req["next_token"] / (req["tokens"] - 1)) + avg_token.append((req["first_token"] + req["next_token"]) / req["tokens"]) + e2e_lat.append(req["first_token"] + req["next_token"]) + tokens += req["tokens"] + + # Statistics for success response data only + if tokens == 0: + req_msg = "Succeed Response: {} (Total {}, {:.1%} Success), Duration: {:.2f}s, RPS: {:.2f}" + else: + req_msg = ( + "Succeed Response: {} (Total {}, {:.1%} Success), Duration: {:.2f}s, Tokens: {}," + " RPS: {:.2f}, Tokens per Second: {:.2f}" + ) + e2e_msg = "End to End latency(ms), P50: {:.2f}, P99: {:.2f}, Avg: {:.2f}" + first_msg = "First token latency(ms), P50: {:.2f}, P99: {:.2f}, Avg: {:.2f}" + next_msg = "Average Next token latency(ms): {:.2f}" + average_msg = "Average token latency(ms) : {:.2f}" + console_logger.warning("\n=================Total statistics=====================") + if tokens == 0: + console_logger.warning( + req_msg.format( + len(reqlist), + environment.runner.stats.num_requests, + len(reqlist) / environment.runner.stats.num_requests, + duration, + len(reqlist) / duration, + ) + ) + else: + console_logger.warning( + req_msg.format( + len(reqlist), + environment.runner.stats.num_requests, + len(reqlist) / environment.runner.stats.num_requests, + duration, + tokens, + len(reqlist) / duration, + tokens / duration, + ) + ) + console_logger.warning( + e2e_msg.format(numpy.percentile(e2e_lat, 50), numpy.percentile(e2e_lat, 99), numpy.average(e2e_lat)) + ) + if tokens != 0: + console_logger.warning( + first_msg.format( + numpy.percentile(first_token, 50), numpy.percentile(first_token, 99), numpy.average(first_token) + ) + ) + console_logger.warning(next_msg.format(numpy.average(next_token))) + console_logger.warning(average_msg.format(numpy.average(avg_token))) + console_logger.warning("======================================================\n\n") + logging.shutdown() diff --git a/evals/benchmark/stresscli/locust/visualqnabench.py b/evals/benchmark/stresscli/locust/visualqnabench.py new file mode 100644 index 00000000..abf23458 --- /dev/null +++ b/evals/benchmark/stresscli/locust/visualqnabench.py @@ -0,0 +1,38 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import os +import random + +import tokenresponse as token + +cwd = os.path.dirname(__file__) +filename = f"{cwd}/../dataset/chatqna.json" +qlist = [] +try: + with open(filename) as qfile: + qlist = json.load(qfile) +except: + logging.error(f"Question File open failed: {filename}") + exit() + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + qid = random.randint(1, 189) + logging.debug(f"Selected question: {qlist[qid]['qText']}") + + return {"messages": qlist[qid]["qText"], "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/locust/visualqnafixed.py b/evals/benchmark/stresscli/locust/visualqnafixed.py new file mode 100644 index 00000000..9ed29e18 --- /dev/null +++ b/evals/benchmark/stresscli/locust/visualqnafixed.py @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import tokenresponse as token + + +def getUrl(): + return "/v1/chatqna" + + +def getReqData(): + return {"messages": "What is the revenue of Nike in last 10 years before 2023? Give me detail", "max_tokens": 128} + + +def respStatics(environment, resp): + return token.respStatics(environment, resp) + + +def staticsOutput(environment, reqlist): + token.staticsOutput(environment, reqlist) diff --git a/evals/benchmark/stresscli/requirements.txt b/evals/benchmark/stresscli/requirements.txt new file mode 100644 index 00000000..e77a6ca5 --- /dev/null +++ b/evals/benchmark/stresscli/requirements.txt @@ -0,0 +1,9 @@ +click +deepdiff +flask +kubernetes +locust +numpy +pytest +pyyaml +requests diff --git a/evals/benchmark/stresscli/run.yaml b/evals/benchmark/stresscli/run.yaml new file mode 100644 index 00000000..6f51dc4f --- /dev/null +++ b/evals/benchmark/stresscli/run.yaml @@ -0,0 +1,27 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +profile: + storage: + hostpath: /home/sdp/test_reports + global-settings: + tool: locust + locustfile: /home/sdp/GenAIEval/evals/benchmark/stresscli/locust/aistress.py + host: http://127.0.0.1:8888 + stop-timeout: 120 + processes: 2 + namespace: default + bench-target: chatqnafixed + run-time: 60m + service-metric-collect: False + service-list: + - "chatqna-tei" + - "chatqna-tgi" + - "chatqna-teirerank" + runs: + - name: sample + users: 2 + max-request: 2 + - name: another + users: 4 + max-request: 4 diff --git a/evals/benchmark/stresscli/stresscli.py b/evals/benchmark/stresscli/stresscli.py new file mode 100755 index 00000000..e4a79494 --- /dev/null +++ b/evals/benchmark/stresscli/stresscli.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from commands.main import cli + +if __name__ == "__main__": + cli()