Skip to content
Snippets Groups Projects
Commit 2106a87c authored by Ivan Pavlovich's avatar Ivan Pavlovich
Browse files

Recalculated tokens and LLM price with real input/output prompts

parent e5e43b0d
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 447 additions and 409 deletions
models/LLM/Ollama/ollama_wrapper.py
+ 1
− 1
View file @ 2106a87c
......@@ -19,7 +19,7 @@ def classify(model, sequence, labels):
prompt += 'text: ' + sequence + '\n'
prompt += 'labels: ' + "[" + ", ".join(labels) + ']\n'
prompt += 'Give the response in json format: { "labels": [] }. Follow this writing to the letter and don t add text around it. Only put the labels that you select between [].'
prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between \'\'.'
prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between "".'
response: ChatResponse = chat(model=model, messages=[
{
......
models/LLM/Tokenizer/data/token_count.json
+ 213
− 213
View file @ 2106a87c
This diff is collapsed.
models/LLM/Tokenizer/token_count.py
+ 16
− 3
View file @ 2106a87c
......@@ -8,7 +8,7 @@ from datetime import datetime, timedelta
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../")))
from testModel.utils import get_article_data
from variables.pubmed import NCDS_MESH_TERM, KEYWORDS_MESH_TERM, KEYWORDS_MESH_SUBHEADING, KEYWORDS_MESH_SITE_PROPOSITION, KEYWORDS_MESH_PROPOSITION
from variables.pubmed import NCDS, NCDS_MESH_TERM, KEYWORDS_MESH_TERM, KEYWORDS_MESH_SUBHEADING, KEYWORDS_MESH_SITE_PROPOSITION, KEYWORDS_MESH_PROPOSITION
DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../dataSources/PubMed/data"))
RESULT_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "./data"))
......@@ -137,7 +137,14 @@ for tokenizer_name in TOKENIZERS:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
output_token_num = 100
output_exemple = '```json\n'
output_exemple += '{'
output_exemple += '\'labels\': [' + ', '.join(NCDS) + ']'
output_exemple += '}'
output_exemple += '```'
output_tokens = tokenizer(output_exemple, return_tensors="pt")
output_token_num = len(output_tokens["input_ids"][0])
i = 1
for article in data:
......@@ -147,7 +154,13 @@ for tokenizer_name in TOKENIZERS:
article_date = datetime(int(article["Date"]["Year"]), int(article["Date"]["Month"]), int(article["Date"]["Day"]))
title, abstract = get_article_data(article)
tokens = tokenizer(title+abstract, return_tensors="pt")
prompt = 'I need you to give me the labels that could be given to the text (keep in mind that u can put multiple labels and select only the labels that i give you):\n'
prompt += 'text: ' + title + abstract + '\n'
prompt += 'labels: ' + "[" + ", ".join(NCDS) + ']\n'
prompt += 'Give the response in json format: { "labels": [] }. Follow this writing to the letter and don t add text around it. Only put the labels that you select between [].'
prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between "".'
tokens = tokenizer(prompt, return_tensors="pt")
num_tokens = len(tokens["input_ids"][0])
add_token_number(article_date, start_date, num_tokens, output_token_num, counts, tokenizer_name, "NO KEYWORDS")
......
models/LLM/prices/data/llm_prices.json
+ 192
− 192
View file @ 2106a87c
{
"Mistral Large": {
"ALL": 495.855152,
"ALL": 702.3351359999999,
"NO KEYWORDS": {
"day": 0.032945036912280706,
"week": 0.23020600326904217,
"month": 1.0017072034139403
"day": 0.03153566877192982,
"week": 0.22035793396534814,
"month": 0.9588547937411096
},
"KEYWORDS": {
"day": 0.001468795602339181,
"week": 0.010263323308270676,
"month": 0.04465932574679943
"day": 0.0014062348538011697,
"week": 0.009826175220660347,
"month": 0.04275714082503557
},
"SUBHEADINGS": {
"day": 0.001468795602339181,
"week": 0.010263323308270676,
"month": 0.04465932574679943
"day": 0.0014062348538011697,
"week": 0.009826175220660347,
"month": 0.04275714082503557
},
"SITE PROPOSITION": {
"day": 0.0019225152748538013,
"week": 0.013433724746649231,
"month": 0.058454856330014224
"day": 0.001841470222222222,
"week": 0.012867416149068325,
"month": 0.05599064864864864
},
"PROPOSITION": {
"day": 0.002632865216374269,
"week": 0.018397350114416476,
"month": 0.08005333428165007
"day": 0.0025214139883040936,
"week": 0.01761857600523047,
"month": 0.07666461450924608
}
},
"Mistral Small": {
"ALL": 24.792757599999998,
"ALL": 35.116756800000005,
"NO KEYWORDS": {
"day": 0.0016472518456140351,
"week": 0.011510300163452107,
"month": 0.050085360170697014
"day": 0.0015767834385964914,
"week": 0.011017896698267408,
"month": 0.04794273968705548
},
"KEYWORDS": {
"day": 7.343978011695905e-05,
"week": 0.0005131661654135338,
"month": 0.0022329662873399716
"day": 7.031174269005849e-05,
"week": 0.0004913087610330174,
"month": 0.002137857041251779
},
"SUBHEADINGS": {
"day": 7.343978011695905e-05,
"week": 0.0005131661654135338,
"month": 0.0022329662873399716
"day": 7.031174269005849e-05,
"week": 0.0004913087610330174,
"month": 0.002137857041251779
},
"SITE PROPOSITION": {
"day": 9.612576374269007e-05,
"week": 0.0006716862373324615,
"month": 0.002922742816500711
"day": 9.20735111111111e-05,
"week": 0.0006433708074534163,
"month": 0.0027995324324324323
},
"PROPOSITION": {
"day": 0.00013164326081871344,
"week": 0.0009198675057208238,
"month": 0.004002666714082503
"day": 0.0001260706994152047,
"week": 0.0008809288002615234,
"month": 0.003833230725462305
}
},
"GPT-4o": {
"ALL": 774.773675,
"ALL": 1097.39865,
"NO KEYWORDS": {
"day": 0.04765905052631579,
"week": 0.3330213157894737,
"month": 1.4490927524893316
"day": 0.04229139040935673,
"week": 0.29551437397842434,
"month": 1.2858868705547655
},
"KEYWORDS": {
"day": 0.0021235734502923977,
"week": 0.014838634357633215,
"month": 0.06456811166429588
"day": 0.0018852869005847954,
"week": 0.013173588591042826,
"month": 0.05732291251778096
},
"SUBHEADINGS": {
"day": 0.0021235734502923977,
"week": 0.014838634357633215,
"month": 0.06456811166429588
"day": 0.0018852869005847954,
"week": 0.013173588591042826,
"month": 0.05732291251778096
},
"SITE PROPOSITION": {
"day": 0.002775600233918129,
"week": 0.019394722131415496,
"month": 0.08439325035561879
"day": 0.00246696,
"week": 0.017238074534161492,
"month": 0.07500891891891892
},
"PROPOSITION": {
"day": 0.003803116608187135,
"week": 0.02657457257273619,
"month": 0.1156353022759602
"day": 0.00337876970760234,
"week": 0.023609415658712,
"month": 0.1027328627311522
}
},
"GPT-4o mini": {
"ALL": 46.486420499999994,
"ALL": 65.843919,
"NO KEYWORDS": {
"day": 0.0028595430315789473,
"week": 0.019981278947368418,
"month": 0.08694556514935989
"day": 0.0025374834245614035,
"week": 0.017730862438705457,
"month": 0.07715321223328592
},
"KEYWORDS": {
"day": 0.00012741440701754387,
"week": 0.0008903180614579927,
"month": 0.003874086699857752
"day": 0.0001131172140350877,
"week": 0.0007904153154625694,
"month": 0.003439374751066857
},
"SUBHEADINGS": {
"day": 0.00012741440701754387,
"week": 0.0008903180614579927,
"month": 0.003874086699857752
"day": 0.0001131172140350877,
"week": 0.0007904153154625694,
"month": 0.003439374751066857
},
"SITE PROPOSITION": {
"day": 0.00016653601403508771,
"week": 0.0011636833278849297,
"month": 0.005063595021337126
"day": 0.00014801759999999998,
"week": 0.0010342844720496896,
"month": 0.004500535135135135
},
"PROPOSITION": {
"day": 0.00022818699649122803,
"week": 0.001594474354364171,
"month": 0.006938118136557611
"day": 0.00020272618245614036,
"week": 0.0014165649395227198,
"month": 0.006163971763869132
}
},
"DeepSeek-V3": {
"ALL": 84.91519478000001,
"ALL": 120.27489204000001,
"NO KEYWORDS": {
"day": 0.005198999491929825,
"week": 0.036328412598888526,
"month": 0.15807768725462307
"day": 0.004590444599766082,
"week": 0.03207608804184374,
"month": 0.13957432904694167
},
"KEYWORDS": {
"day": 0.00023164656421052632,
"week": 0.001618648352402746,
"month": 0.007043307695590327
"day": 0.00020463093192982457,
"week": 0.0014298745243543642,
"month": 0.006221886443812235
},
"SUBHEADINGS": {
"day": 0.00023164656421052632,
"week": 0.001618648352402746,
"month": 0.007043307695590327
"day": 0.00020463093192982457,
"week": 0.0014298745243543642,
"month": 0.006221886443812235
},
"SITE PROPOSITION": {
"day": 0.00030274447438596495,
"week": 0.002115450519777705,
"month": 0.009205068477951637
"day": 0.00026775265777777773,
"week": 0.00187094248447205,
"month": 0.008141128108108108
},
"PROPOSITION": {
"day": 0.00041483287438596496,
"week": 0.002898676917293233,
"month": 0.01261316172119488
"day": 0.00036672314619883045,
"week": 0.002562506456358287,
"month": 0.011150365931721196
}
},
"Gemini 2.0 Flash": {
"ALL": 30.990947000000002,
"ALL": 43.89594600000001,
"NO KEYWORDS": {
"day": 0.0019063620210526318,
"week": 0.013320852631578948,
"month": 0.05796371009957327
"day": 0.0016916556163742691,
"week": 0.011820574959136973,
"month": 0.051435474822190616
},
"KEYWORDS": {
"day": 8.49429380116959e-05,
"week": 0.0005935453743053285,
"month": 0.002582724466571835
"day": 7.541147602339182e-05,
"week": 0.000526943543641713,
"month": 0.002292916500711238
},
"SUBHEADINGS": {
"day": 8.49429380116959e-05,
"week": 0.0005935453743053285,
"month": 0.002582724466571835
"day": 7.541147602339182e-05,
"week": 0.000526943543641713,
"month": 0.002292916500711238
},
"SITE PROPOSITION": {
"day": 0.00011102400935672515,
"week": 0.0007757888852566199,
"month": 0.003375730014224751
"day": 9.86784e-05,
"week": 0.0006895229813664598,
"month": 0.003000356756756757
},
"PROPOSITION": {
"day": 0.0001521246643274854,
"week": 0.0010629829029094476,
"month": 0.0046254120910384075
"day": 0.00013515078830409358,
"week": 0.00094437662634848,
"month": 0.004109314509246088
}
},
"Gemini 2.0 Flash-Lite": {
"ALL": 23.243210249999997,
"ALL": 32.9219595,
"NO KEYWORDS": {
"day": 0.0014297715157894737,
"week": 0.009990639473684209,
"month": 0.043472782574679944
"day": 0.0012687417122807017,
"week": 0.008865431219352729,
"month": 0.03857660611664296
},
"KEYWORDS": {
"day": 6.370720350877193e-05,
"week": 0.00044515903072899637,
"month": 0.001937043349928876
"day": 5.655860701754385e-05,
"week": 0.0003952076577312847,
"month": 0.0017196873755334285
},
"SUBHEADINGS": {
"day": 6.370720350877193e-05,
"week": 0.00044515903072899637,
"month": 0.001937043349928876
"day": 5.655860701754385e-05,
"week": 0.0003952076577312847,
"month": 0.0017196873755334285
},
"SITE PROPOSITION": {
"day": 8.326800701754386e-05,
"week": 0.0005818416639424649,
"month": 0.002531797510668563
"day": 7.400879999999999e-05,
"week": 0.0005171422360248448,
"month": 0.0022502675675675674
},
"PROPOSITION": {
"day": 0.00011409349824561402,
"week": 0.0007972371771820855,
"month": 0.0034690590682788054
"day": 0.00010136309122807018,
"week": 0.0007082824697613599,
"month": 0.003081985881934566
}
},
"Claude 3.7 Sonnet": {
"ALL": 1115.674092,
"ALL": 1580.2540560000002,
"NO KEYWORDS": {
"day": 0.06496416589473684,
"week": 0.4539421529911736,
"month": 1.975261800853485
"day": 0.054195833824561404,
"week": 0.37869759660019614,
"month": 1.6478462987197726
},
"KEYWORDS": {
"day": 0.002893382877192982,
"week": 0.0202177374959137,
"month": 0.08797447937411096
"day": 0.0024153362807017545,
"week": 0.01687734978751226,
"month": 0.07343927880512092
},
"SUBHEADINGS": {
"day": 0.002893382877192982,
"week": 0.0202177374959137,
"month": 0.08797447937411096
"day": 0.0024153362807017545,
"week": 0.01687734978751226,
"month": 0.07343927880512092
},
"SITE PROPOSITION": {
"day": 0.0037776676491228073,
"week": 0.02639674599542334,
"month": 0.11486151635846373
"day": 0.0031584986666666664,
"week": 0.022070254658385094,
"month": 0.09603543243243243
},
"PROPOSITION": {
"day": 0.005178182035087719,
"week": 0.03618294900294214,
"month": 0.15744472403982931
"day": 0.0043269263157894745,
"week": 0.030234733573063097,
"month": 0.13156194879089617
}
},
"Claude 3.5 Haiku": {
"ALL": 297.51309119999996,
"ALL": 421.4010816,
"NO KEYWORDS": {
"day": 0.017323777571929825,
"week": 0.12105124079764629,
"month": 0.526736480227596
"day": 0.014452222353216375,
"week": 0.1009860257600523,
"month": 0.439425679658606
},
"KEYWORDS": {
"day": 0.0007715687672514619,
"week": 0.005391396665576986,
"month": 0.023459861166429587
"day": 0.0006440896748538011,
"week": 0.00450062661000327,
"month": 0.01958380768136558
},
"SUBHEADINGS": {
"day": 0.0007715687672514619,
"week": 0.005391396665576986,
"month": 0.023459861166429587
"day": 0.0006440896748538011,
"week": 0.00450062661000327,
"month": 0.01958380768136558
},
"SITE PROPOSITION": {
"day": 0.0010073780397660819,
"week": 0.007039132265446224,
"month": 0.030629737695590326
"day": 0.000842266311111111,
"week": 0.005885401242236026,
"month": 0.02560944864864865
},
"PROPOSITION": {
"day": 0.0013808485426900583,
"week": 0.00964878640078457,
"month": 0.04198525974395448
"day": 0.0011538470175438598,
"week": 0.008062595619483491,
"month": 0.03508318634423897
}
},
"Claude 3 Opus": {
"ALL": 5578.370459999999,
"ALL": 7901.27028,
"NO KEYWORDS": {
"day": 0.3248208294736842,
"week": 2.2697107649558674,
"month": 9.876309004267425
"day": 0.27097916912280706,
"week": 1.8934879830009808,
"month": 8.239231493598862
},
"KEYWORDS": {
"day": 0.01446691438596491,
"week": 0.10108868747956848,
"month": 0.4398723968705548
"day": 0.012076681403508772,
"week": 0.0843867489375613,
"month": 0.36719639402560456
},
"SUBHEADINGS": {
"day": 0.01446691438596491,
"week": 0.10108868747956848,
"month": 0.4398723968705548
"day": 0.012076681403508772,
"week": 0.0843867489375613,
"month": 0.36719639402560456
},
"SITE PROPOSITION": {
"day": 0.018888338245614034,
"week": 0.1319837299771167,
"month": 0.5743075817923186
"day": 0.01579249333333333,
"week": 0.11035127329192547,
"month": 0.4801771621621621
},
"PROPOSITION": {
"day": 0.025890910175438597,
"week": 0.18091474501471067,
"month": 0.7872236201991465
"day": 0.02163463157894737,
"week": 0.15117366786531547,
"month": 0.6578097439544808
}
},
"Command A": {
"ALL": 774.773675,
"ALL": 1097.39865,
"NO KEYWORDS": {
"day": 0.04765905052631579,
"week": 0.3330213157894737,
"month": 1.4490927524893316
"day": 0.04229139040935673,
"week": 0.29551437397842434,
"month": 1.2858868705547655
},
"KEYWORDS": {
"day": 0.0021235734502923977,
"week": 0.014838634357633215,
"month": 0.06456811166429588
"day": 0.0018852869005847954,
"week": 0.013173588591042826,
"month": 0.05732291251778096
},
"SUBHEADINGS": {
"day": 0.0021235734502923977,
"week": 0.014838634357633215,
"month": 0.06456811166429588
"day": 0.0018852869005847954,
"week": 0.013173588591042826,
"month": 0.05732291251778096
},
"SITE PROPOSITION": {
"day": 0.002775600233918129,
"week": 0.019394722131415496,
"month": 0.08439325035561879
"day": 0.00246696,
"week": 0.017238074534161492,
"month": 0.07500891891891892
},
"PROPOSITION": {
"day": 0.003803116608187135,
"week": 0.02657457257273619,
"month": 0.1156353022759602
"day": 0.00337876970760234,
"week": 0.023609415658712,
"month": 0.1027328627311522
}
},
"Command R+": {
"ALL": 774.773675,
"ALL": 1097.39865,
"NO KEYWORDS": {
"day": 0.04765905052631579,
"week": 0.3330213157894737,
"month": 1.4490927524893316
"day": 0.04229139040935673,
"week": 0.29551437397842434,
"month": 1.2858868705547655
},
"KEYWORDS": {
"day": 0.0021235734502923977,
"week": 0.014838634357633215,
"month": 0.06456811166429588
"day": 0.0018852869005847954,
"week": 0.013173588591042826,
"month": 0.05732291251778096
},
"SUBHEADINGS": {
"day": 0.0021235734502923977,
"week": 0.014838634357633215,
"month": 0.06456811166429588
"day": 0.0018852869005847954,
"week": 0.013173588591042826,
"month": 0.05732291251778096
},
"SITE PROPOSITION": {
"day": 0.002775600233918129,
"week": 0.019394722131415496,
"month": 0.08439325035561879
"day": 0.00246696,
"week": 0.017238074534161492,
"month": 0.07500891891891892
},
"PROPOSITION": {
"day": 0.003803116608187135,
"week": 0.02657457257273619,
"month": 0.1156353022759602
"day": 0.00337876970760234,
"week": 0.023609415658712,
"month": 0.1027328627311522
}
}
}
\ No newline at end of file
testModel/__pycache__/metrics.cpython-313.pyc
+ 0
− 0
View file @ 2106a87c
No preview for this file type
testModel/__pycache__/utils.cpython-313.pyc
+ 0
− 0
View file @ 2106a87c
No preview for this file type
testModel/launch_tests.py 0 → 100644
+ 25
− 0
View file @ 2106a87c
import subprocess
import sys
import os
import argparse
# Ajouter le répertoire parent au chemin de recherche
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../")))
from variables.models import MODELS
TEST_SCRIPT_FILEPATH = "./test.py"
file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), TEST_SCRIPT_FILEPATH))
args = ["-m"]
try:
result = subprocess.run(["python", file_path], check=True, capture_output=True, text=True)
print("Script output:", result.stdout)
except subprocess.CalledProcessError as e:
print("Script crashed with error:", e.stderr)
except FileNotFoundError:
print("Python executable or script not found.")
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment