File size: 5,258 Bytes
eec7921 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 |
import functools
import gradio as gr
import pandas as pd
import numpy as np
from typing import List
_ORIGINAL_DF = pd.read_csv('./data/benchmark.csv')
_METRICS = {'MCC', 'F1', 'ACC'}
_AGGREGATION_METHODS = {'mean', 'max', 'min', 'median'}
_DATASETS = set(_ORIGINAL_DF['Dataset'])
_BIBTEX = """@article{DallaTorre2023TheNT,
title={The Nucleotide Transformer: Building and Evaluating Robust Foundation Models for Human Genomics},
author={Hugo Dalla-Torre and Liam Gonzalez and Javier Mendoza Revilla and Nicolas Lopez Carranza and Adam Henryk Grzywaczewski and Francesco Oteri and Christian Dallago and Evan Trop and Hassan Sirelkhatim and Guillaume Richard and Marcin J. Skwark and Karim Beguir and Marie Lopez and Thomas Pierrot},
journal={bioRxiv},
year={2023},
url={https://api.semanticscholar.org/CorpusID:255943445}
}
"""
_LAST_UPDATED = 'Aug 28, 2023'
banner_url = "./assets/logo.png"
_BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>'
_INTRODUCTION_TEXT = "The 🤗 Nucleotide Transformer Leaderboard aims to track, rank and evaluate DNA foundational models on a set of curated downstream tasks with a standardized evaluation protocole."
def retrieve_array_from_text(text):
return np.fromstring(text.replace('[', '').replace(']', ''), dtype=float, sep=',')
def format_number(x):
return float(f'{x:.3}')
def get_dataset(tasks: List[str], target_metric: str = 'MCC', aggregation_method: str = 'mean'):
aggr_fn = getattr(np, aggregation_method)
scores = _ORIGINAL_DF[target_metric].apply(retrieve_array_from_text).apply(aggr_fn)
scores = scores.apply(format_number)
df = _ORIGINAL_DF.drop(columns=list(_METRICS))
df['Score'] = scores
df = df.pivot(index='Model', columns='Dataset', values='Score')
df = df[tasks]
df['All Tasks'] = df.agg('mean', axis='columns').apply(format_number)
columns = list(df.columns.values)
columns.sort()
df = df[columns]
df.reset_index(inplace=True)
df = df.rename(columns={'index': 'Model'})
df = df.sort_values(by=['All Tasks'], ascending=False)
leaderboard_table = gr.components.Dataframe.update(
value=df,
# datatype=TYPES,
max_rows=None,
interactive=False,
visible=True,
)
return leaderboard_table
with gr.Blocks() as demo:
with gr.Row():
gr.Image(banner_url, height=160, scale=1)
gr.Textbox(_INTRODUCTION_TEXT, scale=5)
with gr.Row():
metric_choice = gr.Dropdown(
choices=list(_METRICS),
value="MCC",
label="Metric displayed.",
)
aggr_choice = gr.Dropdown(
choices=list(_AGGREGATION_METHODS),
value="mean",
label="Aggregation used over 10-folds.",
)
with gr.Row():
selected_tasks = gr.CheckboxGroup(
choices=list(_DATASETS),
value=list(_DATASETS),
label="Tasks",
info="Downstream tasks."
)
with gr.Tabs(elem_classes="tab-buttons") as tabs:
with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
dataframe = gr.components.Dataframe(elem_id="leaderboard-table",)
with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1):
gr.Markdown('Hey hey hey', elem_classes="markdown-text")
# with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table",
# id=2):
# with gr.Column():
# gr.Markdown("# ✉️✨ Request results for a new model here!",
# elem_classes="markdown-text")
# with gr.Column():
# gr.Markdown("Select a dataset:", elem_classes="markdown-text")
# with gr.Column():
# model_name_textbox = gr.Textbox(
# label="Model name (user_name/model_name)")
# chb_coco2017 = gr.Checkbox(label="COCO validation 2017 dataset",
# visible=False, value=True,
# interactive=False)
# with gr.Column():
# mdw_submission_result = gr.Markdown()
# btn_submitt = gr.Button(value="🚀 Request")
gr.Markdown(f"Last updated on **{_LAST_UPDATED}**", elem_classes="markdown-text")
with gr.Row():
with gr.Accordion("📙 Citation", open=False):
gr.Textbox(
value=_BIBTEX, lines=7,
label="Copy the BibTeX snippet to cite this source",
elem_id="citation-button",
).style(show_copy_button=True)
selected_tasks.change(get_dataset, inputs=[selected_tasks, metric_choice, aggr_choice], outputs=dataframe)
metric_choice.change(get_dataset, inputs=[selected_tasks, metric_choice, aggr_choice], outputs=dataframe)
aggr_choice.change(get_dataset, inputs=[selected_tasks, metric_choice, aggr_choice], outputs=dataframe)
demo.load(fn=get_dataset, inputs=[selected_tasks, metric_choice, aggr_choice], outputs=dataframe)
demo.launch()
|