sync escada : gestion fine des pendings + détection orphelines
- importer.py : nouvelle logique pour les 4 cas d'absence × pending :
* abs en PDF + pending modify : pending wins (sans force) / override (force)
* abs en PDF + pas en DB + pending action=clear : respecte la suppression
locale (sans force) / recrée l'abs (force)
* orpheline (DB sans PDF) sans pending : supprimée + comptée + détaillée
* orpheline avec pending : conservée (sans force) / supprimée (force)
- importer.py : query orpheline par classe + fenêtre de dates du PDF
(couvre les abs locales avec import_id=None)
- run_imports.py : remonte orphelines + pending_skipped dans res_abs
- notifier.py : niveau detailed inclut absences supprimées par classe
+ détail des orphelines (max 5 par classe)
- escada.py : sépare cache disque (toutes classes pour matching Matu)
vs liste UI (filtrée MP/MI/Formation)
- escada.py : timeout polling import passe de 60s à 15min
- escada.py : retire mode test push, fix bouton Actualiser bloqué sans
classe sélectionnée
- cron.py : reset last_run_at à l'activation d'un job pour relance
immédiate au prochain tick
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
4d3e49ff64
commit
f60cbf1b1c
9 changed files with 166 additions and 70 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -14,3 +14,6 @@ data/pdfs/
|
|||
data/sync_*.json
|
||||
data/debug_*.png
|
||||
data/*.bak.*
|
||||
|
||||
# Logs cron (runtime)
|
||||
logs/
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
{
|
||||
"AUTOMAT 1:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=687fa97d-1032-4078-94ae-1899fc1e6014",
|
||||
"AUTOMAT 2:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=8cb48a35-290c-4488-b98c-437d2c9186a6",
|
||||
"EM-AU 1:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=da3e0b68-5559-4c0c-a8be-f764c68dbca9",
|
||||
"EM-AU 2:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=ef32322a-8bd9-45d4-9583-c7c22cbc577d"
|
||||
"AUTOMAT 1:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=5aae2a26-4117-4722-aee2-0bd823fbcfc8",
|
||||
"AUTOMAT 2:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=23fb9e1d-adda-4399-accd-7a2f49e0cc93",
|
||||
"AUTOMAT 3:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=4587e860-dc1f-496a-84a6-dbf1f5d0a963",
|
||||
"AUTOMAT 4:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=4d128fbe-f18c-4e3b-9ec2-beea462837be",
|
||||
"EM-AU 3:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=30066257-3dad-4b00-857b-9ab60a5d8581",
|
||||
"EM-AU 4:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=b9363b96-2d6e-4009-a495-f26c036cc088",
|
||||
"MONTAUT 1:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=b51ec970-5bf4-4982-a05e-80546bb7421f",
|
||||
"MONTAUT 2:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=aaacc343-c248-4f21-93f6-5d9e3079aa5d",
|
||||
"MONTAUT 3:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=18c1ddbe-471f-44f6-bde6-8619adc3b767",
|
||||
"EM-AU 2:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=c2c982af-b6f9-44ab-8ed5-7bc567b967cf",
|
||||
"EM-AU 1:abs": "https://escadaweb.vs.ch/Lehrpersonen/ViewAbsenzenErweitert.aspx?id=a4c4c187-920c-4c91-9620-7f153cf3738a"
|
||||
}
|
||||
|
|
@ -23,6 +23,8 @@ services:
|
|||
- API_URL=https://dev.dashboard.eptm-automation.ch
|
||||
# Évite la boucle infinie de hot-reload causée par SQLite WAL/SHM dans data/
|
||||
- REFLEX_HOT_RELOAD_EXCLUDE_PATHS=/app/data
|
||||
# Timezone du container : aligne avec le host (cohérence cron + logs)
|
||||
- TZ=Europe/Zurich
|
||||
networks:
|
||||
- default
|
||||
- proxy_net
|
||||
|
|
|
|||
|
|
@ -405,8 +405,13 @@ class CronState(AuthState):
|
|||
try:
|
||||
job = sess.get(CronJob, job_id)
|
||||
if job:
|
||||
was_disabled = not job.enabled
|
||||
job.enabled = not job.enabled
|
||||
job.updated_at = datetime.now()
|
||||
# Si on passe de désactivé → activé, reset last_run_at pour qu'il
|
||||
# tourne au prochain tick (au lieu d'attendre last_run_at + interval).
|
||||
if was_disabled and job.enabled:
|
||||
job.last_run_at = None
|
||||
sess.commit()
|
||||
self._refresh()
|
||||
finally:
|
||||
|
|
|
|||
|
|
@ -81,7 +81,6 @@ class EscadaState(AuthState):
|
|||
push_done: bool = False
|
||||
push_ok: int = 0
|
||||
push_errors: list[str] = []
|
||||
push_test: bool = False
|
||||
|
||||
@rx.var
|
||||
def selected_count(self) -> int:
|
||||
|
|
@ -120,7 +119,6 @@ class EscadaState(AuthState):
|
|||
def set_sync_notes(self, v: bool): self.sync_notes = v
|
||||
def set_sync_fiches(self, v: bool): self.sync_fiches = v
|
||||
def set_force_abs(self, v: bool): self.force_abs = v
|
||||
def set_push_test(self, v: bool): self.push_test = v
|
||||
|
||||
def _clear_results(self):
|
||||
self.sync_done = False
|
||||
|
|
@ -176,14 +174,23 @@ class EscadaState(AuthState):
|
|||
if CLASSES_CACHE.exists():
|
||||
try:
|
||||
cached = json.loads(CLASSES_CACHE.read_text(encoding="utf-8"))
|
||||
# Filtrer MP/MI (formations maturité, hors scope)
|
||||
cached = [c for c in cached if c and not c.startswith(("MP", "MI"))]
|
||||
self.classes_cache = cached
|
||||
for c in cached:
|
||||
# Le fichier cache contient TOUTES les classes (y compris MP/MI/
|
||||
# Formation) — utilisé par sync_esacada.py pour le matching Matu.
|
||||
# Pour l'affichage UI, on filtre.
|
||||
ui_classes = [
|
||||
c for c in cached
|
||||
if c and not c.startswith(("MP", "MI")) and c != "Formation"
|
||||
]
|
||||
self.classes_cache = ui_classes
|
||||
for c in ui_classes:
|
||||
if c not in self.class_checked:
|
||||
self.class_checked[c] = False
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# Cache file absent — reset state mémoire (clé pour vider via UI)
|
||||
self.classes_cache = []
|
||||
self.class_checked = {}
|
||||
self._reload_pending()
|
||||
# Vider les résultats à chaque visite de la page
|
||||
self.sync_done = False
|
||||
|
|
@ -301,17 +308,28 @@ class EscadaState(AuthState):
|
|||
|
||||
new_classes: list[str] = []
|
||||
for line in lines:
|
||||
if line.startswith("CLASSES_JSON:"):
|
||||
# Le script préfixe parfois ses lignes avec un timestamp [HH:MM:SS],
|
||||
# donc on cherche la sous-chaîne au lieu de checker startswith.
|
||||
idx = line.find("CLASSES_JSON:")
|
||||
if idx >= 0:
|
||||
try:
|
||||
new_classes = json.loads(line[len("CLASSES_JSON:"):])
|
||||
new_classes = json.loads(line[idx + len("CLASSES_JSON:"):])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Filtrer MP/MI (formations maturité, hors scope)
|
||||
new_classes = [c for c in new_classes if c and not c.startswith(("MP", "MI"))]
|
||||
# Garder la liste BRUTE (toutes classes y compris MP/MI/Formation)
|
||||
# pour le cache disque : sync_esacada.py s'en sert pour trouver les
|
||||
# classes Matu (MP1-TASV) correspondant aux années des classes sélectionnées.
|
||||
all_classes = [c for c in new_classes if c]
|
||||
|
||||
if new_classes:
|
||||
app_log(f"Classes recuperees : {', '.join(new_classes)}")
|
||||
# Filtrer MP/MI/Formation pour l'affichage UI (multi-select)
|
||||
ui_classes = [
|
||||
c for c in all_classes
|
||||
if not c.startswith(("MP", "MI")) and c != "Formation"
|
||||
]
|
||||
|
||||
if ui_classes:
|
||||
app_log(f"Classes recuperees : {', '.join(ui_classes)}")
|
||||
else:
|
||||
app_log(f"Aucune classe recuperee (code={_rc_holder[0]}, lignes={len(lines)})")
|
||||
|
||||
|
|
@ -321,13 +339,14 @@ class EscadaState(AuthState):
|
|||
for _ in range(_t.cancelling()):
|
||||
_t.uncancel()
|
||||
async with self:
|
||||
if new_classes:
|
||||
self.classes_cache = new_classes
|
||||
if ui_classes:
|
||||
self.classes_cache = ui_classes
|
||||
existing = dict(self.class_checked)
|
||||
self.class_checked = {c: existing.get(c, False) for c in new_classes}
|
||||
self.class_checked = {c: existing.get(c, False) for c in ui_classes}
|
||||
try:
|
||||
# Cache disque : liste COMPLÈTE pour le matching Matu
|
||||
CLASSES_CACHE.write_text(
|
||||
json.dumps(new_classes, ensure_ascii=False), encoding="utf-8"
|
||||
json.dumps(all_classes, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -550,10 +569,12 @@ class EscadaState(AuthState):
|
|||
self.is_syncing = False
|
||||
self.import_in_progress = True
|
||||
|
||||
# Polling inline (max 20 × 3s = 60s) — aucune mise à jour d'état ici
|
||||
# Polling inline (max 300 × 3s = 15 min) — aucune mise à jour d'état ici
|
||||
# Un import complet (toutes classes + BN + Matu + Notes via Selenium)
|
||||
# peut facilement prendre plusieurs minutes.
|
||||
_result_data: dict = {}
|
||||
_result_ready = False
|
||||
for _ in range(20):
|
||||
for _ in range(300):
|
||||
try:
|
||||
await asyncio.sleep(3)
|
||||
except asyncio.CancelledError:
|
||||
|
|
@ -582,22 +603,21 @@ class EscadaState(AuthState):
|
|||
self.sync_done = True
|
||||
app_log("Resultats charges — sync terminee OK")
|
||||
else:
|
||||
self.sync_errors = ["Import timeout — verifiez les logs (> 60s)."]
|
||||
self.sync_errors = ["Import timeout — verifiez les logs (> 15min)."]
|
||||
|
||||
# ── Background: push vers Escada ───────────────────────────────────────────
|
||||
|
||||
@_background
|
||||
async def push_escada(self):
|
||||
async with self:
|
||||
push_test = self.push_test
|
||||
self.is_pushing = True
|
||||
self.op_log = "Envoi vers Escadaweb..."
|
||||
self.push_done = False
|
||||
self.push_ok = 0
|
||||
self.push_errors = []
|
||||
|
||||
app_log(f"Push Escada demarre (test={push_test})")
|
||||
extra = ["--test"] if push_test else []
|
||||
app_log("Push Escada demarre")
|
||||
extra: list[str] = []
|
||||
cmd = [sys.executable, str(_PUSH_SCRIPT), *extra]
|
||||
lines: list[str] = []
|
||||
_rc_holder = [0]
|
||||
|
|
@ -1069,7 +1089,7 @@ def escada_page() -> rx.Component:
|
|||
),
|
||||
" Actualiser",
|
||||
on_click=EscadaState.refresh_classes,
|
||||
disabled=EscadaState.sync_disabled,
|
||||
disabled=EscadaState.is_busy,
|
||||
variant="outline",
|
||||
color_scheme="gray",
|
||||
size="1",
|
||||
|
|
@ -1235,15 +1255,6 @@ def escada_page() -> rx.Component:
|
|||
),
|
||||
|
||||
rx.flex(
|
||||
rx.flex(
|
||||
rx.checkbox(
|
||||
checked=EscadaState.push_test,
|
||||
on_change=EscadaState.set_push_test,
|
||||
size="2",
|
||||
),
|
||||
rx.text("Mode test", size="2", color="#555"),
|
||||
gap="0.4rem", align="center",
|
||||
),
|
||||
rx.button(
|
||||
rx.cond(
|
||||
EscadaState.is_pushing,
|
||||
|
|
@ -1293,12 +1304,6 @@ def escada_page() -> rx.Component:
|
|||
),
|
||||
),
|
||||
|
||||
# Log push
|
||||
rx.cond(
|
||||
~EscadaState.is_pushing,
|
||||
_log_box(),
|
||||
),
|
||||
|
||||
padding="1.25rem",
|
||||
background_color="white",
|
||||
border_radius="8px",
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
[cron_tick] 2026-05-10T09:29:02 — 1 job(s) dûs
|
||||
- #3 'Import absences toutes les 2h' kind=sync schedule=interval:120
|
||||
|
|
@ -53,6 +53,8 @@ for pdf_path in abs_pdfs:
|
|||
detail += f", {r.nb_absences_mises_a_jour} maj"
|
||||
if r.nb_absences_pending_skipped:
|
||||
detail += f", {r.nb_absences_pending_skipped} pending"
|
||||
if r.nb_absences_supprimees:
|
||||
detail += f", {r.nb_absences_supprimees} orphelines"
|
||||
res_abs.append({
|
||||
"classe": r.classe,
|
||||
"detail": detail,
|
||||
|
|
@ -60,8 +62,14 @@ for pdf_path in abs_pdfs:
|
|||
"mises_a_jour": r.nb_absences_mises_a_jour,
|
||||
"pending_skipped": r.nb_absences_pending_skipped,
|
||||
"doublons": r.nb_absences_doublons,
|
||||
"orphelines": r.nb_absences_supprimees,
|
||||
"pendings_orphelins": r.nb_pendings_orphelins_supprimes,
|
||||
"details_orphelines": r.details_orphelines,
|
||||
})
|
||||
app_log(f"[run_imports] abs {r.classe}: {detail}")
|
||||
if r.details_orphelines:
|
||||
for d in r.details_orphelines:
|
||||
app_log(f"[run_imports] orpheline supprimée : {r.classe} | {d}")
|
||||
except Exception as e:
|
||||
errors.append(f"Import abs {Path(pdf_path).name}: {e}")
|
||||
app_log(f"[run_imports] erreur abs: {e}")
|
||||
|
|
|
|||
|
|
@ -23,10 +23,12 @@ class ImportResult:
|
|||
nb_absences_nouvelles: int
|
||||
nb_absences_doublons: int
|
||||
nb_absences_mises_a_jour: int = 0
|
||||
nb_absences_supprimees: int = 0
|
||||
nb_absences_pending_skipped: int = 0 # absences non modifiées car pending vers Escada
|
||||
nb_absences_supprimees: int = 0 # orphelines : abs en DB plus présentes dans le PDF Escada
|
||||
nb_pendings_orphelins_supprimes: int = 0 # pendings supprimés en cascade avec orphelines
|
||||
nb_absences_pending_skipped: int = 0 # abs non modifiées car pending modif locale
|
||||
details_nouvelles: list[str] = field(default_factory=list)
|
||||
details_mises_a_jour: list[str] = field(default_factory=list)
|
||||
details_orphelines: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def import_pdf(
|
||||
|
|
@ -89,15 +91,15 @@ def import_pdf(
|
|||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existe is not None:
|
||||
ep_pending = session.execute(
|
||||
select(EscadaPending).where(
|
||||
EscadaPending.apprenti_id == existe.apprenti_id,
|
||||
EscadaPending.date == existe.date,
|
||||
EscadaPending.periode == existe.periode,
|
||||
EscadaPending.apprenti_id == apprenti.id,
|
||||
EscadaPending.date == ab["date"],
|
||||
EscadaPending.periode == ab["periode"],
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existe is not None:
|
||||
if force:
|
||||
existe.type_origine = ab["type_absence"]
|
||||
existe.statut = "excusee" if ab["type_absence"] == "E" else "a_traiter"
|
||||
|
|
@ -130,6 +132,16 @@ def import_pdf(
|
|||
else:
|
||||
nb_doublons += 1
|
||||
else:
|
||||
# Cas: l'absence est dans le PDF Escada mais pas en DB.
|
||||
# Si un pending existe (typiquement action="clear" = suppression
|
||||
# locale en attente de push), respecter le pending sauf en force.
|
||||
if ep_pending and not force:
|
||||
nb_pending_skipped += 1
|
||||
continue
|
||||
if ep_pending and force:
|
||||
# Force : on écrase le pending et on (ré)insère l'absence
|
||||
session.delete(ep_pending)
|
||||
|
||||
statut_initial = "excusee" if ab["type_absence"] == "E" else "a_traiter"
|
||||
session.add(
|
||||
Absence(
|
||||
|
|
@ -150,13 +162,33 @@ def import_pdf(
|
|||
except Exception:
|
||||
entry["dates"].append(str(ab["date"]))
|
||||
|
||||
# ── Diff : absences supprimées dans Escada ────────────────────────────────
|
||||
# ── Diff : absences supprimées dans Escada (orphelines) ──────────────────
|
||||
# On veut toutes les abs des apprentis de cette classe dans la fenêtre de
|
||||
# dates couverte par le PDF — y compris les abs créées localement
|
||||
# (import_id=None) qui ne joignent pas Import.
|
||||
pdf_dates: set = set()
|
||||
for a_data in apprentis_data:
|
||||
for ab in a_data["absences"]:
|
||||
pdf_dates.add(ab["date"])
|
||||
if pdf_dates:
|
||||
date_min = min(pdf_dates)
|
||||
date_max = max(pdf_dates)
|
||||
prev_absences = session.execute(
|
||||
select(Absence)
|
||||
.join(Import, Absence.import_id == Import.id)
|
||||
.where(Import.classe == classe, Import.semestre == semestre)
|
||||
.join(Apprenti, Apprenti.id == Absence.apprenti_id)
|
||||
.where(
|
||||
Apprenti.classe == classe,
|
||||
Absence.date >= date_min,
|
||||
Absence.date <= date_max,
|
||||
)
|
||||
).scalars().all()
|
||||
else:
|
||||
prev_absences = []
|
||||
|
||||
details_orphelines: list[str] = []
|
||||
nb_pendings_orphelins = 0
|
||||
# Index des apprentis pour récupérer leur nom/prénom sans re-query par orphelin
|
||||
apprenti_by_id: dict[int, Apprenti] = {}
|
||||
for ab in prev_absences:
|
||||
if (ab.apprenti_id, ab.date, ab.periode) not in seen_keys:
|
||||
ep = session.execute(
|
||||
|
|
@ -166,8 +198,31 @@ def import_pdf(
|
|||
EscadaPending.periode == ab.periode,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
# Orphelin avec pending : préserve l'intent local (sauf force)
|
||||
# — typique : utilisateur a ajouté localement (action="E") et n'a pas
|
||||
# encore pushé vers Escada. Supprimer écraserait son travail.
|
||||
if ep and not force:
|
||||
nb_pending_skipped += 1
|
||||
continue
|
||||
|
||||
if ep:
|
||||
session.delete(ep)
|
||||
nb_pendings_orphelins += 1
|
||||
|
||||
ap = apprenti_by_id.get(ab.apprenti_id)
|
||||
if ap is None:
|
||||
ap = session.get(Apprenti, ab.apprenti_id)
|
||||
if ap is not None:
|
||||
apprenti_by_id[ab.apprenti_id] = ap
|
||||
|
||||
try:
|
||||
date_str = ab.date.strftime("%d.%m.%Y")
|
||||
except Exception:
|
||||
date_str = str(ab.date)
|
||||
nom = f"{ap.prenom} {ap.nom}" if ap else f"apprenti#{ab.apprenti_id}"
|
||||
details_orphelines.append(f"{nom} — {date_str} P{ab.periode}")
|
||||
|
||||
session.delete(ab)
|
||||
nb_supprimees += 1
|
||||
|
||||
|
|
@ -191,7 +246,9 @@ def import_pdf(
|
|||
nb_absences_doublons=nb_doublons,
|
||||
nb_absences_mises_a_jour=nb_mises_a_jour,
|
||||
nb_absences_supprimees=nb_supprimees,
|
||||
nb_pendings_orphelins_supprimes=nb_pendings_orphelins,
|
||||
nb_absences_pending_skipped=nb_pending_skipped,
|
||||
details_nouvelles=[_fmt(d) for d in _nouv_by_ap.values()],
|
||||
details_mises_a_jour=[_fmt(d) for d in _upd_by_ap.values()],
|
||||
details_orphelines=details_orphelines,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -144,10 +144,21 @@ def notify_job_result(
|
|||
nouv = int(r.get("nouvelles", 0) or 0)
|
||||
maj = int(r.get("mises_a_jour", 0) or 0)
|
||||
pend = int(r.get("pending_skipped", 0) or 0)
|
||||
parts.append(
|
||||
orph = int(r.get("orphelines", 0) or 0)
|
||||
line = (
|
||||
f" • <b>{_escape_html(classe)}</b> : "
|
||||
f"{nouv} nouv. · {maj} modif. · {pend} pending"
|
||||
)
|
||||
if orph:
|
||||
line += f" · <b>{orph} suppr.</b>"
|
||||
parts.append(line)
|
||||
# Liste des orphelines (cap à 5 par classe pour limiter la taille)
|
||||
details_orph = r.get("details_orphelines") or []
|
||||
if details_orph:
|
||||
for d in details_orph[:5]:
|
||||
parts.append(f" ↳ {_escape_html(str(d))}")
|
||||
if len(details_orph) > 5:
|
||||
parts.append(f" ↳ … +{len(details_orph) - 5} autre(s)")
|
||||
|
||||
# BN (seulement si sync_bn coché)
|
||||
if job_options.get("sync_bn"):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue