| \n", + " | raw_prompt | \n", + "prediction | \n", + "first_line_prediction | \n", + "first_line_groundtruth | \n", + "first_line_matched | \n", + "first_statement_prediction | \n", + "first_statement_groundtruth | \n", + "first_statement_matched | \n", + "edit_distance | \n", + "
|---|---|---|---|---|---|---|---|---|---|
| 2568 | \n", + "from classes import BaseModule, Response, MDEM... | \n", + "create_html_table(DetailedResults['Accounts'],... | \n", + "create_html_table(DetailedResults['Accounts'],... | \n", + "list_to_html_table(linked_accounts_list, escap... | \n", + "False | \n", + "create_html_table(DetailedResults['Accounts'],... | \n", + "list_to_html_table(linked_accounts_list, escap... | \n", + "False | \n", + "476.0 | \n", + "
| 471 | \n", + "import os\\nimport time\\n\\nimport torch as th\\n... | \n", + "ModelType.ModelType.ModelType.ModelType.ModelT... | \n", + "ModelType.ModelType.ModelType.ModelType.ModelT... | \n", + "DEIR and not self.use_model_rnn: | \n", + "False | \n", + "ModelType.ModelType.ModelType.ModelType.ModelT... | \n", + "DEIR and not self.use_model_rnn: | \n", + "False | \n", + "441.0 | \n", + "
| 2406 | \n", + "import agents.utils.basicprompts as p\\n\\n\\nsys... | \n", + "ask_question(question=system_init, prompt=\"Wha... | \n", + "ask_question(question=system_init, prompt=\"Wha... | \n", + "prompting_utils + p.using_steps + reevaluateAt... | \n", + "False | \n", + "ask_question(question=system_init, prompt=\"Wha... | \n", + "prompting_utils + p.using_steps + reevaluateAt... | \n", + "False | \n", + "424.0 | \n", + "
| 2332 | \n", + "import openai\\nfrom .backend import Backend\\n\\... | \n", + "model_name, max_length=self.max_length, temper... | \n", + "model_name, max_length=self.max_length, temper... | \n", + "model_name)['data'][0]['embedding'] | \n", + "False | \n", + "model_name, max_length=self.max_length, temper... | \n", + "model_name)['data'][0]['embedding'] | \n", + "False | \n", + "385.0 | \n", + "
| 807 | \n", + "\\nimport os\\nfrom argparse import ArgumentPars... | \n", + "num_joints, data_module.num_joints, data_modul... | \n", + "num_joints, data_module.num_joints, data_modul... | \n", + "skeleton, window, 1.0 / frequency,batch_size=b... | \n", + "False | \n", + "num_joints, data_module.num_joints, data_modul... | \n", + "skeleton, window, 1.0 / frequency,batch_size=b... | \n", + "False | \n", + "382.0 | \n", + "
| ... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "
| 213 | \n", + "#! /usr/bin/env python\\n# coding=utf8\\n\\nimpor... | \n", + "add_GrpcServiceServicer_to_server(TestService(... | \n", + "add_GrpcServiceServicer_to_server(TestService(... | \n", + "add_GrpcServiceServicer_to_server(TestService(... | \n", + "False | \n", + "add_GrpcServiceServicer_to_server(TestService(... | \n", + "add_GrpcServiceServicer_to_server(TestService(... | \n", + "False | \n", + "1.0 | \n", + "
| 1978 | \n", + "\"\"\"A package for decoding and encoding each it... | \n", + "get_buffer())\\n self.file_desc.clos... | \n", + "get_buffer()) | \n", + "getbuffer()) | \n", + "False | \n", + "get_buffer()) | \n", + "getbuffer()) | \n", + "False | \n", + "1.0 | \n", + "
| 1319 | \n", + "import threading\\nimport time\\nfrom agency.pro... | \n", + "process()\\n\\n self.__thread = threading... | \n", + "process() | \n", + "_process() | \n", + "False | \n", + "process() | \n", + "_process() | \n", + "False | \n", + "1.0 | \n", + "
| 826 | \n", + "import asyncio\\nimport sys\\nimport os\\nsys.pat... | \n", + "delete_old_resources_snapshots(time)\\n\\n ... | \n", + "delete_old_resources_snapshots(time) | \n", + "delete_old_resource_snapshots(time) | \n", + "False | \n", + "delete_old_resources_snapshots(time) | \n", + "delete_old_resource_snapshots(time) | \n", + "False | \n", + "1.0 | \n", + "
| 1984 | \n", + "from functools import lru_cache\\nfrom typing i... | \n", + "headers, **kwargs)\\n return [completion... | \n", + "headers, **kwargs) | \n", + "_headers, **kwargs) | \n", + "False | \n", + "headers, **kwargs) | \n", + "_headers, **kwargs) | \n", + "False | \n", + "1.0 | \n", + "
2635 rows × 9 columns
\n", + "import faiss
import numpy as np
import json
from .device import Device
import os
from pathlib import Path
class VectorIndexedDB(Device):
def __init__(self, embedding_backend, **kwargs):
super().__init__(**kwargs)
self.embedding_backend = embedding_backend
self.index = faiss.IndexFlatL2(embedding_backend.dim)
self.embeddings = []
self.texts = []
async def asyncAdd(self, text, caller):
\"\"\"
Adds text to the database, which can be queried later.
\"\"\"
embedding = await self.embedding_backend.asyncRequest(caller, text)
embedding = np.array(embedding)[None]
self.embeddings.append(embedding)
self.texts.append(text)
self.index.add(embedding)
return f\"Added a new item to {self.
| t | 1 | return f\"Added a new item to {self.<end of prompt>name}.\" | t | 1 | return f\"Added a new item to {self.<end of prompt>name}. \" |
| Legends | |||||||||
|---|---|---|---|---|---|---|---|---|---|
| \n",
+ "
|
{prompt}
\"\n", + "html_file = prompt_part + html_file\n", + "IPython.display.HTML(data=html_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "659dbd07-1cdf-44be-9aa9-e025e1a476e9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python/tabby-eval/pyproject.toml b/python/tabby-eval/pyproject.toml new file mode 100644 index 000000000000..abe8303c76f4 --- /dev/null +++ b/python/tabby-eval/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "tabby_data_pipeline" diff --git a/python/tabby-eval/setup.cfg b/python/tabby-eval/setup.cfg new file mode 100644 index 000000000000..5df12ba8b1e4 --- /dev/null +++ b/python/tabby-eval/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +name = tabby_data_pipeline diff --git a/python/tabby-eval/setup.py b/python/tabby-eval/setup.py new file mode 100644 index 000000000000..22b3230dac2a --- /dev/null +++ b/python/tabby-eval/setup.py @@ -0,0 +1,17 @@ +from setuptools import find_packages, setup + +setup( + name="tabby_data_pipeline", + packages=find_packages(exclude=["tabby_data_pipeline_tests"]), + install_requires=[ + "dagster", + "dagster-cloud", + "dagstermill", + "papermill-origami>=0.0.8", + "pandas", + "matplotlib", + "seaborn", + "scikit-learn", + ], + extras_require={"dev": ["dagster-webserver", "pytest"]}, +) diff --git a/python/tabby-eval/tabby_data_pipeline/__init__.py b/python/tabby-eval/tabby_data_pipeline/__init__.py new file mode 100644 index 000000000000..09b810fff658 --- /dev/null +++ b/python/tabby-eval/tabby_data_pipeline/__init__.py @@ -0,0 +1,17 @@ +from dagster import Definitions, load_assets_from_modules + +from dagstermill import ConfigurableLocalOutputNotebookIOManager + + +from . import assets, create_csv + +all_assets = load_assets_from_modules([assets, create_csv]) + +defs = Definitions( + assets=all_assets, + resources = { + "output_notebook_io_manager": ConfigurableLocalOutputNotebookIOManager() + } +) + + diff --git a/python/tabby-eval/tabby_data_pipeline/analyze.py b/python/tabby-eval/tabby_data_pipeline/analyze.py new file mode 100644 index 000000000000..f7f80aa48a7d --- /dev/null +++ b/python/tabby-eval/tabby_data_pipeline/analyze.py @@ -0,0 +1,78 @@ +import json + + +def get_bracket_lang_statement(completion): + end_idx = None + for i in range(len(completion)): + if completion[i] in [";", "{", "}"]: + end_idx = i + break + return completion[:end_idx+1] if end_idx else completion + + +def postprocess_code_lines(prompt, target, language): + try: + if language in ["java", "csharp", "typescript"]: + return get_bracket_lang_statement(target) + elif language == "python": + return target.split("\n")[0] + except Exception: + return target + + +def analyze(model, language, file): + + line_match = 0 + statement_match = 0 + + input_file = f"./data/{model}/{language}/{file}" + output_file = f"./data/{model}/{language}/result_{file}" + + with open(output_file, 'w') as fout: + with open(input_file) as fin: + for line in fin: + obj = json.loads(line) + result = {} + prediction = "" + + for k in obj.keys(): + if k == "prediction": + prediction = str(obj[k]) + break + elif k == "error": + break + else: + result[k] = obj[k] + + tabby_eval = {} + if file == "line_completion.jsonl": + tabby_eval["raw_prompt"] = obj["prompt"] + else: + tabby_eval["raw_prompt"] = obj["crossfile_context"]["text"] + obj["prompt"] + + tabby_eval["prediction"] = prediction + + groundtruth = obj["groundtruth"] + + tabby_eval["first_line_prediction"] = prediction.split("\n")[0] + tabby_eval["first_line_groundtruth"] = groundtruth.split("\n")[0] + if tabby_eval["first_line_prediction"] == tabby_eval["first_line_groundtruth"]: + tabby_eval["first_line_matched"] = True + line_match += 1 + else: + tabby_eval["first_line_matched"] = False + + tabby_eval["first_statement_prediction"] = postprocess_code_lines(tabby_eval["raw_prompt"], prediction, language) + tabby_eval["first_statement_groundtruth"] = postprocess_code_lines(tabby_eval["raw_prompt"], groundtruth, language) + if tabby_eval["first_statement_prediction"] == tabby_eval["first_statement_groundtruth"]: + tabby_eval["first_statement_matched"] = True + statement_match += 1 + else: + tabby_eval["first_statement_matched"] = False + + result["tabby_eval"] = tabby_eval + + json.dump(result, fout) + fout.write("\n") + + diff --git a/python/tabby-eval/tabby_data_pipeline/assets.py b/python/tabby-eval/tabby_data_pipeline/assets.py new file mode 100644 index 000000000000..5ebceb4896bd --- /dev/null +++ b/python/tabby-eval/tabby_data_pipeline/assets.py @@ -0,0 +1,145 @@ +import os +import subprocess + +from dagster import ( + AssetExecutionContext, + MetadataValue, + asset, + StaticPartitionsDefinition, + MultiPartitionsDefinition, +) +from . import analyze + + +@asset +def baseline() -> str: + return "line_completion.jsonl" + +@asset +def bm25() -> str: + return "line_completion_rg1_bm25.jsonl" + +@asset +def oracle() -> str: + return "line_completion_oracle_bm25.jsonl" + +@asset( + partitions_def=MultiPartitionsDefinition( + { + "model_id" : StaticPartitionsDefinition(['TabbyML/StarCoder-1B', 'TabbyML/StarCoder-3B', 'TabbyML/StarCoder-7B', 'TabbyML/WizardCoder-1B', 'TabbyML/WizardCoder-3B', 'TabbyML/CodeLlama-7B', 'TabbyML/CodeLlama-13B']), + "language" : StaticPartitionsDefinition(["python", "java", "csharp", "typescript"]), + + } + )) +def predict_baseline(context: AssetExecutionContext, baseline: str) -> None: + model_id = context.partition_key.keys_by_dimension["model_id"] + language = context.partition_key.keys_by_dimension["language"] + + my_env = os.environ.copy() + my_env["MODEL_ID"] = model_id + + context.add_output_metadata(metadata={"model_id": MetadataValue.md(model_id)}) + + files = baseline + + p = subprocess.Popen(["modal", "run", "./modal/predict.py","--language", language, "--files", files], env=my_env) + p.wait() + context.add_output_metadata(metadata={'modal run': MetadataValue.md("success!")}) + +@asset( + partitions_def=MultiPartitionsDefinition( + { + "model_id" : StaticPartitionsDefinition(['TabbyML/StarCoder-1B', 'TabbyML/StarCoder-3B', 'TabbyML/StarCoder-7B', 'TabbyML/WizardCoder-1B', 'TabbyML/WizardCoder-3B', 'TabbyML/CodeLlama-7B', 'TabbyML/CodeLlama-13B']), + "language" : StaticPartitionsDefinition(["python", "java", "csharp", "typescript"]), + + } + )) +def predict_bm25(context: AssetExecutionContext, bm25: str) -> None: + model_id = context.partition_key.keys_by_dimension["model_id"] + language = context.partition_key.keys_by_dimension["language"] + + my_env = os.environ.copy() + my_env["MODEL_ID"] = model_id + + context.add_output_metadata(metadata={"model_id": MetadataValue.md(model_id)}) + + files = bm25 + + p = subprocess.Popen(["modal", "run", "./modal/predict.py","--language", language, "--files", files], env=my_env) + p.wait() + context.add_output_metadata(metadata={'modal run': MetadataValue.md("success!")}) + + +@asset( + partitions_def=MultiPartitionsDefinition( + { + "model_id" : StaticPartitionsDefinition(['TabbyML/StarCoder-1B', 'TabbyML/StarCoder-3B', 'TabbyML/StarCoder-7B', 'TabbyML/WizardCoder-1B', 'TabbyML/WizardCoder-3B', 'TabbyML/CodeLlama-7B', 'TabbyML/CodeLlama-13B']), + "language" : StaticPartitionsDefinition(["python", "java", "csharp", "typescript"]), + + } + )) +def predict_oracle(context: AssetExecutionContext, oracle: str) -> None: + model_id = context.partition_key.keys_by_dimension["model_id"] + language = context.partition_key.keys_by_dimension["language"] + + my_env = os.environ.copy() + my_env["MODEL_ID"] = model_id + + context.add_output_metadata(metadata={"model_id": MetadataValue.md(model_id)}) + + files = oracle + + p = subprocess.Popen(["modal", "run", "./modal/predict.py","--language", language, "--files", files], env=my_env) + p.wait() + context.add_output_metadata(metadata={'modal run': MetadataValue.md("success!")}) + + + +@asset( + partitions_def=MultiPartitionsDefinition( + { + "model_id" : StaticPartitionsDefinition(['TabbyML/StarCoder-1B', 'TabbyML/StarCoder-3B', 'TabbyML/StarCoder-7B', 'TabbyML/WizardCoder-1B', 'TabbyML/WizardCoder-3B', 'TabbyML/CodeLlama-7B', 'TabbyML/CodeLlama-13B']), + "language" : StaticPartitionsDefinition(["python", "java", "csharp", "typescript"]), + } + ), deps=[predict_baseline]) +def matching_baseline(context) -> None: + model_id = context.partition_key.keys_by_dimension["model_id"] + language = context.partition_key.keys_by_dimension["language"] + + + model = model_id.split("/")[-1] + analyze.analyze(model, language, 'line_completion.jsonl') + + + +@asset( + partitions_def=MultiPartitionsDefinition( + { + "model_id" : StaticPartitionsDefinition(['TabbyML/StarCoder-1B', 'TabbyML/StarCoder-3B', 'TabbyML/StarCoder-7B', 'TabbyML/WizardCoder-1B', 'TabbyML/WizardCoder-3B', 'TabbyML/CodeLlama-7B', 'TabbyML/CodeLlama-13B']), + "language" : StaticPartitionsDefinition(["python", "java", "csharp", "typescript"]), + } + ), deps=[predict_bm25]) +def matching_bm25(context) -> None: + model_id = context.partition_key.keys_by_dimension["model_id"] + language = context.partition_key.keys_by_dimension["language"] + + + model = model_id.split("/")[-1] + analyze.analyze(model, language, 'line_completion_rg1_bm25.jsonl') + + + +@asset( + partitions_def=MultiPartitionsDefinition( + { + "model_id" : StaticPartitionsDefinition(['TabbyML/StarCoder-1B', 'TabbyML/StarCoder-3B', 'TabbyML/StarCoder-7B', 'TabbyML/WizardCoder-1B', 'TabbyML/WizardCoder-3B', 'TabbyML/CodeLlama-7B', 'TabbyML/CodeLlama-13B']), + "language" : StaticPartitionsDefinition(["python", "java", "csharp", "typescript"]), + } + ), deps=[predict_oracle]) +def matching_oracle(context) -> None: + model_id = context.partition_key.keys_by_dimension["model_id"] + language = context.partition_key.keys_by_dimension["language"] + + + model = model_id.split("/")[-1] + analyze.analyze(model, language, 'line_completion_oracle_bm25.jsonl') \ No newline at end of file diff --git a/python/tabby-eval/tabby_data_pipeline/create_csv.py b/python/tabby-eval/tabby_data_pipeline/create_csv.py new file mode 100644 index 000000000000..10b46cde71b8 --- /dev/null +++ b/python/tabby-eval/tabby_data_pipeline/create_csv.py @@ -0,0 +1,55 @@ +import json +import pandas as pd + +from dagster import ( + asset, + AssetIn, + file_relative_path + ) +from dagstermill import define_dagstermill_asset + + + +models = ["StarCoder-1B", "StarCoder-3B", "StarCoder-7B", "CodeLlama-7B", "CodeLlama-13B", "WizardCoder-1B", "WizardCoder-3B", "DeepseekCoder-1.3B", "DeepseekCoder-6.7B"] +languages = {"csharp": "C#", "java": "Java", "python": "Python", "typescript": "Typescript"} +files = ["line_completion.jsonl", 'line_completion_rg1_bm25.jsonl', 'line_completion_oracle_bm25.jsonl'] +total_records = {'python': 2665, 'java': 2139, 'typescript': 3356, 'csharp': 1768} + +headers = ['Model', 'Dataset', 'Records', 'baseline', 'bm25', 'oracle'] + +stat = [] +def get_match(model, language, file): + count = 0 + with open(f"./data/{model}/{language}/result_{file}") as f: + for line in f: + obj = json.loads(line) + if obj["tabby_eval"]["first_line_matched"]: + count += 1 + + return count + +@asset +def create_csv(): + for model in models: + for language in languages.keys(): + x = [model, languages[language], total_records[language]] + for f in files: + x.append(get_match(model, language, f)) + + stat.append(x) + + df = pd.DataFrame(stat, columns=headers) + print(df) + + df.to_csv('./tabby_data_pipeline/tabby.csv', index=False) + + +@asset(deps=[create_csv]) +def tabby_dataset(): + return pd.read_csv(file_relative_path(__file__,'tabby.csv')) + +tabby_jupyter_notebook = define_dagstermill_asset( + name = 'tabby_jupyter', + notebook_path = file_relative_path(__file__, "tabby_eval.ipynb"), + ins={"df": AssetIn("tabby_dataset")}, +) diff --git a/python/tabby-eval/tabby_data_pipeline/tabby.csv b/python/tabby-eval/tabby_data_pipeline/tabby.csv new file mode 100644 index 000000000000..998ff3b7063e --- /dev/null +++ b/python/tabby-eval/tabby_data_pipeline/tabby.csv @@ -0,0 +1,37 @@ +Model,Dataset,Records,baseline,bm25,oracle +StarCoder-1B,C#,1768,31,100,138 +StarCoder-1B,Java,2139,17,81,98 +StarCoder-1B,Python,2665,30,237,338 +StarCoder-1B,Typescript,3356,144,270,342 +StarCoder-3B,C#,1768,51,118,165 +StarCoder-3B,Java,2139,39,93,114 +StarCoder-3B,Python,2665,130,322,445 +StarCoder-3B,Typescript,3356,229,351,422 +StarCoder-7B,C#,1768,64,134,170 +StarCoder-7B,Java,2139,60,116,138 +StarCoder-7B,Python,2665,176,376,494 +StarCoder-7B,Typescript,3356,267,391,457 +CodeLlama-7B,C#,1768,63,133,181 +CodeLlama-7B,Java,2139,82,150,175 +CodeLlama-7B,Python,2665,193,394,488 +CodeLlama-7B,Typescript,3356,483,689,766 +CodeLlama-13B,C#,1768,84,152,197 +CodeLlama-13B,Java,2139,99,154,182 +CodeLlama-13B,Python,2665,208,411,538 +CodeLlama-13B,Typescript,3356,525,733,869 +WizardCoder-1B,C#,1768,30,87,138 +WizardCoder-1B,Java,2139,25,77,99 +WizardCoder-1B,Python,2665,37,218,343 +WizardCoder-1B,Typescript,3356,159,264,338 +WizardCoder-3B,C#,1768,51,118,165 +WizardCoder-3B,Java,2139,39,93,114 +WizardCoder-3B,Python,2665,142,334,446 +WizardCoder-3B,Typescript,3356,234,348,422 +DeepseekCoder-1.3B,C#,1768,58,121,166 +DeepseekCoder-1.3B,Java,2139,54,115,136 +DeepseekCoder-1.3B,Python,2665,149,332,433 +DeepseekCoder-1.3B,Typescript,3356,437,590,708 +DeepseekCoder-6.7B,C#,1768,87,166,198 +DeepseekCoder-6.7B,Java,2139,83,150,165 +DeepseekCoder-6.7B,Python,2665,247,436,571 +DeepseekCoder-6.7B,Typescript,3356,555,740,857 diff --git a/python/tabby-eval/tabby_data_pipeline/tabby_eval.ipynb b/python/tabby-eval/tabby_data_pipeline/tabby_eval.ipynb new file mode 100644 index 000000000000..f9f5155416e7 --- /dev/null +++ b/python/tabby-eval/tabby_data_pipeline/tabby_eval.ipynb @@ -0,0 +1,370 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "slide" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "df = pd.read_csv('tabby.csv')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "df['baseline_percent'] = round(df[\"baseline\"]/df[\"Records\"]*100, 2)\n", + "df['bm25_percent'] = round(df[\"bm25\"]/df[\"Records\"]*100, 2)\n", + "df['oracle_percent'] = round(df['oracle']/df[\"Records\"]*100, 2)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "| \n", + " | Model | \n", + "Dataset | \n", + "Records | \n", + "baseline | \n", + "bm25 | \n", + "oracle | \n", + "baseline_percent | \n", + "bm25_percent | \n", + "oracle_percent | \n", + "
|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "StarCoder-1B | \n", + "C# | \n", + "1768 | \n", + "31 | \n", + "100 | \n", + "138 | \n", + "1.75 | \n", + "5.66 | \n", + "7.81 | \n", + "
| 1 | \n", + "StarCoder-1B | \n", + "Java | \n", + "2139 | \n", + "17 | \n", + "81 | \n", + "98 | \n", + "0.79 | \n", + "3.79 | \n", + "4.58 | \n", + "
| 2 | \n", + "StarCoder-1B | \n", + "Python | \n", + "2665 | \n", + "30 | \n", + "237 | \n", + "338 | \n", + "1.13 | \n", + "8.89 | \n", + "12.68 | \n", + "
| 3 | \n", + "StarCoder-1B | \n", + "Typescript | \n", + "3356 | \n", + "144 | \n", + "270 | \n", + "342 | \n", + "4.29 | \n", + "8.05 | \n", + "10.19 | \n", + "
| 4 | \n", + "StarCoder-3B | \n", + "C# | \n", + "1768 | \n", + "51 | \n", + "118 | \n", + "165 | \n", + "2.88 | \n", + "6.67 | \n", + "9.33 | \n", + "