denis/PlanarianScanner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

export-all-verif

Browse Source
This commit is contained in:
denis defolie
2026-05-19 10:53:51 +02:00
parent 5477de46fe
commit 308ddaa048
14 changed files with 345 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test_tube_scanner/planarian/export_service.py
+68
View File
@@ -0,0 +1,68 @@
import logging
import asyncio
from asgiref.sync import async_to_sync
from django.conf import settings
from modules.planarian_metrics import ReductStoreClient
logger = logging.getLogger(__name__)
def _get_reduct_client() -> ReductStoreClient:
"""Instancie le client ReductStore depuis les settings Django."""
return ReductStoreClient(url=settings.REDUCTSTORE_URL, token=settings.REDUCTSTORE_TOKEN)
def export_csv_sync(
*,
experiment,
well,
uuid,
planarian,
record_type,
start=None,
stop=None,
):
#@async_to_sync
async def _run():
client = _get_reduct_client()
await client.connect()
return await client.export_csv_response(
experiment=experiment,
well=well,
uuid=uuid,
planarian=planarian,
record_type=record_type,
start=start,
stop=stop,
)
#return _run()
return asyncio.run(_run())
def export_csv_file_sync(
*,
experiment,
well,
uuid,
planarian,
record_type,
):
async def _run():
client = _get_reduct_client()
await client.connect()
try:
return await client.export_csv(
experiment=experiment,
well=well,
uuid=uuid,
planarian=planarian,
record_type=record_type,
output_dir=settings.CSV_EXPORT_DIR,
)
finally:
await client.close()
return asyncio.run(_run())
test_tube_scanner/planarian/models.py
+2 -1
View File
@@ -143,7 +143,8 @@ class ExperimentConfig(models.Model):
def get_session(self):
return self.experiment_key.session_experiments.first() if self.experiment_key else None
se = self.experiment_key.session_experiments.first() if self.experiment_key else None
return se.session
def to_params_dict(self) -> dict:
test_tube_scanner/planarian/tasks.py
+61 -35
View File
@@ -1,54 +1,80 @@
# tasks.py
from celery import shared_task
from celery import shared_task, group
from celery.utils.log import get_task_logger
from django.conf import settings
from asgiref.sync import async_to_sync
from modules.planarian_metrics import ReductStoreClient
from scanner.models import SessionExperiment
from scanner.models import SessionExperiment, get_uuid_from_session, Experiment
from .models import ExperimentConfig
from .export_service import export_csv_file_sync
logger = get_task_logger(__name__)
def _get_reduct_client() -> ReductStoreClient:
"""Instancie le client ReductStore depuis les settings Django."""
return ReductStoreClient(url=settings.REDUCTSTORE_URL, token=settings.REDUCTSTORE_TOKEN)
@shared_task(bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={"max_retries": 3},)
def export_single_metric(self, *, experiment, well, uuid, planarian, record_type,):
"""
Exporte UN fichier CSV.
"""
logger.info( f"Export start {experiment} {well} {planarian} {record_type}")
f, n = export_csv_file_sync(
experiment=experiment,
well=well,
uuid=uuid,
planarian=planarian,
record_type=record_type,
)
logger.info( f"Export done {f} ({n} rows)" )
return {"file": f, "rows": n, }
@shared_task
def export_experiment_metrics(experiment):
def export_experiment_metrics_task(experiment_id):
experiment = Experiment.objects.filter(pk=experiment_id).first()
if not experiment:
return
@async_to_sync
async def _do_export(well, planarian, record_type):
client = _get_reduct_client()
await client.connect()
try:
f, n = await client.export_csv(
experiment = experiment.identifier,
well = well,
planarian = planarian,
record_type = record_type,
output_dir = settings.CSV_EXPORT_DIR,
)
logger.warning(f"Export CSV {settings.CSV_EXPORT_DIR}/{f} done, {n} lignes")
except Exception as e:
logger.error(f"Erreur export CSV: {e}")
experiment_configs = ExperimentConfig.objects.filter(experiment_key_id=experiment.id).order_by('well').all()
for conf in experiment_configs:
jobs = []
configs = (ExperimentConfig.objects.filter(experiment_key_id=experiment.id).order_by("well"))
for conf in configs:
well = conf.well
count = conf.planarian_count
session = conf.get_session()
uuid = get_uuid_from_session(session.id, conf.experiment_key.multiwell.position, well, )
for record_type in ["frame", "summary"]:
for planarian in range(count):
_do_export(well, planarian, record_type)
'''
jobs.append(
export_single_metric.s(
experiment=experiment.identifier,
well=well,
uuid=uuid,
planarian=planarian,
record_type=record_type,
)
)
'''
try:
f, n = export_csv_file_sync(
experiment=experiment.identifier,
well=well,
uuid=uuid,
planarian=planarian,
record_type=record_type,
)
logger.info( f"Export done {f} ({n} rows)" )
except Exception as e:
logger.exception(e)
'''
result = group(jobs).apply_async()
logger.info(f"{len(jobs)} exports launched group_id={result.id}")
return {"group_id": result.id,"jobs": len(jobs), }'''
@shared_task
def export_session_metrics(session):
experiments = SessionExperiment.experiment_by_session(session.id)
def export_session_metrics_task(session_id):
experiments = SessionExperiment.experiment_by_session(session_id, active=False)
for experiment in experiments:
export_experiment_metrics(experiment)
export_experiment_metrics_task.delay(experiment.id)
test_tube_scanner/planarian/templates/planarian/base.html
+7
View File
@@ -27,6 +27,12 @@
<i class="fa-solid fa-file-export w3-text-red w3-xlarge"></i> {% trans "Exporter les metrics de l'expérience" %}<br>
<span class="w3-margin-left-5">&nbsp;$> {{ export_csv_destination }}
</a>
<a href="#" class="w3-bar-item w3-btn w3-hover-opacity" onclick="export_csv('{{ url_export }}', {{ current_session.id }}, 'session_csv')">
<i class="fa-solid fa-file-export w3-text-pink w3-xxlarge"></i> {% trans "Exporter les metrics de la session" %}<br>
<span class="w3-margin-left-5">&nbsp;$> {{ export_csv_destination }}
</a>
{% endif %}
{% endblock %}
{% endblock %}
@@ -34,6 +40,7 @@
{% block js_footer %}
{{ block.super }}
<script>
const export_msg = "{% trans "Confirmer l'export des vidéos. ATTENTION le processus peut être long!" %}";
// Auto-dismiss messages after 5 seconds
document.addEventListener('DOMContentLoaded', function() {
const messages = document.querySelectorAll('.alert');
test_tube_scanner/planarian/templates/planarian/export_csv.html
+1
View File
@@ -21,6 +21,7 @@
<input type="hidden" id="_sid" name="_sid" value="{{ current_session.id }}">
<input type="hidden" id="_expid" name="_expid" value="{{ current_experiment.id }}">
<div class="w3-card-4 w3-border w3-round w3-round-large">
<header class="w3-container w3-blue w3-round w3-round-top-large">
<h3 class="w3-text-white">{% trans "Paramètres d'export" %}</h3>
test_tube_scanner/planarian/views.py
+35 -46
View File
@@ -20,7 +20,9 @@ from django.views import View
from modules.planarian_metrics import ExperimentParams, ReductStoreClient
from modules.system_stats import get_cached_stats, start_background_updater
from scanner.constants import ScannerConstants
from .tasks import export_experiment_metrics, export_session_metrics
from .tasks import export_experiment_metrics_task, export_session_metrics_task
from .export_service import export_csv_sync
from scanner import models, views as scanner_views
from .models import ExperimentConfig
@@ -82,7 +84,6 @@ def get_active_session(request, session_id=None, experiment_id=None):
# ---------------------------------------------------------------------------
# Vue :Export CSV depuis ReductStore
# ---------------------------------------------------------------------------
@login_required
@csrf_exempt
def export_metrics(request):
@@ -92,48 +93,13 @@ def export_metrics(request):
if action=='experiment_csv':
experiment = models.Experiment.objects.filter(pk=pid).first()
if experiment:
export_experiment_metrics(experiment)
return JsonResponse({"state": True})
export_experiment_metrics_task.delay(experiment.id) # @UndefinedVariable
return JsonResponse({"success": True, "msg": str(_("Métrics en cours de téléchargement"))})
if action=='session_csv':
session = models.Session.objects.filter(pk=pid).first()
if session:
export_session_metrics(session)
return JsonResponse({"state": True})
return JsonResponse({"state": False})
def export_csv(request):
d = request.POST
print("export_csv===========", d)
@async_to_sync
async def _do_export():
client = _get_reduct_client()
await client.connect()
try:
csv_content, n = await client.export_csv_response(
experiment = d.get("experiment"),
well = d.get("well"),
planarian = d.get("planarian"),
record_type = d.get("record_type"),
start = d.get("start_dt"),
stop = d.get("stop_dt"),
)
print(f"Export CSV: export_csv_response done, {n} lignes, content size={len(csv_content)}")
except Exception as e:
logger.error(f"Erreur export CSV: {e}")
messages.error(request, _("Erreur lors de l'export CSV: %(error)s") % {"error": str(e)})
return None, 0
return csv_content, n
csv_content, n = _do_export()
logger.info(f"Export CSV: {n} lignes, content size={len(csv_content)}")
filename = (
f"{d['experiment']}_{d['well']}-planaire-{d['planarian']}"
f"_{d['record_type']}.csv"
)
return csv_content, filename
if pid:
export_session_metrics_task(pid) # @UndefinedVariable
return JsonResponse({"success": True, "msg": str(_("Métrics en cours de téléchargement"))})
return JsonResponse({"success": False, "msg": str(_("Erreur: les métrics non pas été téléchargés"))})
@login_required
@@ -147,12 +113,35 @@ def export_csv_view(request):
**session_context
}
if request.method == 'POST':
valid = request.POST.get('valid')
if valid == 'ok':
csv_content, filename = export_csv(request)
valid = request.POST.get('valid')
session = session_context['current_session']
experiment = session_context['current_experiment']
if valid == 'ok' and session and experiment:
well_name = request.POST.get('well')
uuid = models.get_uuid_from_session(session.id, experiment.multiwell.position, well_name)
'''
csv_content, filename = export_csv(request, uuid)
if csv_content:
response = FileResponse(csv_content, content_type="text/csv; charset=utf-8")
response["Content-Disposition"] = f'attachment; filename="{filename}"'
return response'''
csv_content, n = export_csv_sync(
experiment=experiment.identifier,
well=well_name,
uuid=uuid,
planarian=request.POST.get("planarian"),
record_type=request.POST.get("record_type"),
start=request.POST.get("start_dt"),
stop=request.POST.get("stop_dt"),
)
if csv_content:
filename = (
f"{experiment.identifier}_{well_name}-{request.POST.get('planarian')}_"
f"{request.POST.get('record_type')}.csv"
)
logger.info(f"Export CSV: {n} lignes, content size={len(csv_content)}")
response = FileResponse(csv_content, content_type="text/csv; charset=utf-8")
response["Content-Disposition"] = (f'attachment; filename="{filename}"')
return response
messages.warning(request, _("Aucune donnée trouvée."))