From 16e5f2ced2d9cfd4904d6c0f595e5773aa93b11e Mon Sep 17 00:00:00 2001 From: Jake Kasper Date: Fri, 24 Apr 2026 12:33:52 -0400 Subject: [PATCH] added multi node functionality --- app/__pycache__/__init__.cpython-314.pyc | Bin 0 -> 114 bytes app/__pycache__/config.cpython-314.pyc | Bin 2360 -> 3354 bytes app/config.py | 15 ++ .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 122 bytes .../__pycache__/actions.cpython-314.pyc | Bin 0 -> 1006 bytes .../__pycache__/alerts.cpython-314.pyc | Bin 0 -> 1118 bytes .../emulated_session.cpython-314.pyc | Bin 0 -> 1332 bytes .../__pycache__/network.cpython-314.pyc | Bin 0 -> 568 bytes app/routers/__pycache__/query.cpython-314.pyc | Bin 0 -> 2206 bytes app/routers/alerts.py | 9 +- app/routers/network.py | 7 +- app/routers/query.py | 8 +- .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 123 bytes app/services/__pycache__/ai.cpython-314.pyc | Bin 19424 -> 24317 bytes .../__pycache__/alertmanager.cpython-314.pyc | Bin 0 -> 3045 bytes .../cluster_inventory.cpython-314.pyc | Bin 0 -> 11066 bytes .../__pycache__/log_analyzer.cpython-314.pyc | Bin 18969 -> 19936 bytes app/services/__pycache__/pls.cpython-314.pyc | Bin 0 -> 5598 bytes .../__pycache__/prometheus.cpython-314.pyc | Bin 0 -> 3397 bytes app/services/ai.py | 81 ++++++-- app/services/alertmanager.py | 20 +- app/services/cluster_inventory.py | 180 ++++++++++++++++++ app/services/log_analyzer.py | 16 +- app/services/pls.py | 78 ++++++++ app/services/prometheus.py | 13 +- app/ui/actions.html | 7 + app/ui/index.html | 151 ++++++++++++++- app/ui/tasks.html | 166 ++++++++++------ config/marvis.env.example | 5 + config/p5g-marvis.service | 10 + 30 files changed, 673 insertions(+), 93 deletions(-) create mode 100644 app/__pycache__/__init__.cpython-314.pyc create mode 100644 app/routers/__pycache__/__init__.cpython-314.pyc create mode 100644 app/routers/__pycache__/actions.cpython-314.pyc create mode 100644 app/routers/__pycache__/alerts.cpython-314.pyc create mode 100644 app/routers/__pycache__/emulated_session.cpython-314.pyc create mode 100644 app/routers/__pycache__/network.cpython-314.pyc create mode 100644 app/routers/__pycache__/query.cpython-314.pyc create mode 100644 app/services/__pycache__/__init__.cpython-314.pyc create mode 100644 app/services/__pycache__/alertmanager.cpython-314.pyc create mode 100644 app/services/__pycache__/cluster_inventory.cpython-314.pyc create mode 100644 app/services/__pycache__/pls.cpython-314.pyc create mode 100644 app/services/__pycache__/prometheus.cpython-314.pyc create mode 100644 app/services/cluster_inventory.py create mode 100644 app/services/pls.py diff --git a/app/__pycache__/__init__.cpython-314.pyc b/app/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d148390d5bf7b7bc8012f0f7c61a1ef6b5812251 GIT binary patch literal 114 zcmdPq>P{wCAAftgHh(Vb_lhJP_LlF~@{~08Ci$AfTKtDb{ pGcU6wK3=b&@)n0pZhlH>PO4oID^MO}LNSQ(iJ6g+v4|PS0suZh6w3er literal 0 HcmV?d00001 diff --git a/app/__pycache__/config.cpython-314.pyc b/app/__pycache__/config.cpython-314.pyc index deb9faf8adca25cbb68c04f49f6b8238cd95c703..7ac8b77b60942ea14720fa952c8f505ba19a4446 100644 GIT binary patch delta 1502 zcmaKqO>7%Q6vtWZgR5CJ9Tm?)O;0Iw|2I zLUg$Wt6#e*PF8^9&ijG549 zzzZ;LLVpXoF2YL&OuSOdN&SvDU&jgujDR4#Yy{}NwO^#MGR(ppya6{M0#TTU1&G07 zNYT$u5LjBLdLPhH5{y5^i(1>ELa}4Bi7u|0Jij5NLQ>?mbnR0@b_E+CZ_&FH=$4o{ zX&`O2ZkvdwaB7FjviQB_DNb@rF1@dD`>vKrm$EvyF-TEW(L%h_Hl z-t^28G(9tOH%7jYvV!C^$Jzs1hF-2UY_nG87kk$x?aSfj<(%C0nH&50-BMPYV;M1v z&#m~E(1uJY{eyJ;V-t_J;jR5>k1k)S&uks}wkytKU(Z+ZFXQ#WD~BUTzS;i+(}x$2 be9<~_|L!|iAH4cQ3FDO~&+Kt(!58u$IR0A0 delta 616 zcmYk2OG`pQ6o6;0@2lRtnq^j&k4mkyNQ5Ab+7{=K^-6AlR>$PZAiwJ6!%r5#7 zL7Ubs+PDf@#O)VE8=*z3pqX@r2If1Q?>u1UHvH&UCKa!Y==r|>JbZFAm5~2cLg*ZA zBfu-jKqa&(HE)8d1eP2vuwzJF+1&DBR&1XA#j#G(>PXcCUNPa|YZT}a(#fR{R3Tkl z`XL~s%w;bGg>-Y-2W}xfT=qjqNH3QI5EfG5au9}u?CHw6VTf?K8SOm#KXcs#3iIqk z)b3bx1V;Io;fE;1APxx_gK?OEB%~ls%9cbZs~CX{g5m}?dI85#YVm{ZTPOFnOY&l$ z#^~Y4UdDFQYUmA(RGh7niDylzS}`rBT3zp_%_)x_ra>QKh%k&Y#269`;|xiLG{Y1_ zhGB*x6{Oqhru{%Ow8}BLbLB}y*04%0 diff --git a/app/config.py b/app/config.py index 04ed460..26a89fe 100644 --- a/app/config.py +++ b/app/config.py @@ -1,11 +1,26 @@ import os + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.lower() in {"1", "true", "yes", "on"} + + # Defaults assume the appliance-style deployment model where Marvis runs with # host networking and talks to sibling services over host loopback. PROMETHEUS_URL = os.getenv("MARVIS_PROMETHEUS_URL", "http://127.0.0.1:9090") PROMETHEUS_PREFIX = os.getenv("MARVIS_PROMETHEUS_PREFIX", "/prometheus") ALERTMANAGER_URL = os.getenv("MARVIS_ALERTMANAGER_URL", "http://127.0.0.1:9093") +# PLS discovery defaults assume the local appliance exposes PLS via Traefik. +PLS_BASE_URL = os.getenv("MARVIS_PLS_BASE_URL", "https://127.0.0.1/core/pls/api/1") +PLS_USERNAME = os.getenv("MARVIS_PLS_USERNAME", "") +PLS_PASSWORD = os.getenv("MARVIS_PLS_PASSWORD", "") +PLS_AUTH_BACKEND = os.getenv("MARVIS_PLS_AUTH_BACKEND", "local") +PLS_VERIFY_TLS = _env_bool("MARVIS_PLS_VERIFY_TLS", False) + # AI backend: "rule" | "openai" | "ollama" AI_MODE = os.getenv("MARVIS_AI_MODE", "rule") OPENAI_API_KEY = os.getenv("MARVIS_OPENAI_API_KEY", "") diff --git a/app/routers/__pycache__/__init__.cpython-314.pyc b/app/routers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fbfa5a582f971fca386b41cadcfd73103f64ead GIT binary patch literal 122 zcmdPq>P{wCAAftgHh(Vb_lhJP_LlF~@{~08COFXflK))!z xv?R5tSU)~KGcU6wK3=b&@)n0pZhlH>PO4oID^MNCq+$@`6Ehh;tr|UlGYr~1l=J_2$u>{I!Y7|6UAQQMmMt3HC9fA8pr;85^rV0 zo>{epv|Eg;Ulz=$)ha++FIa_&(bz(x>3BMn98Q9(Iu{!X-)%TMZk^uavyfH)6Y&T_ zU&k09qpQd0`q#;scME$V=t&E|O}lW$ IsnAgWA2P||qyPW_ literal 0 HcmV?d00001 diff --git a/app/routers/__pycache__/alerts.cpython-314.pyc b/app/routers/__pycache__/alerts.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e3f4e5a4dfee818e9733dda34d272d14d958d13 GIT binary patch literal 1118 zcmZ`&O-vI(6n?YY-EIlLtw_pW3m8HXi$qK?Aq_sCXV`AO{Pek|~H0iv&0pri7Sii7}Vu3cx(+g5h$wCCy9mI_Hw=s*1yy zfe+QyQjK=2QV)dWW9|qM=849q)a;t2(~!VwwX{yHRV||}>y$Mm(`zO=7E_sZJ!56* z<^%Vj9bqQATCs31ypOF1Qk}0_Xf#{(^3r>Fef=-DrtdP zXslH3urY`#BtRAC+)m^yI6l>tAgVN{IG(uOZD&_n|DS!()dOyrzJb$yRZNkPJd-T9 z&_3!ydj)4Pfu=+>0TZ17_nYE>$JxQy_G(#V?3q$v%2xuaktj%23lx0)hra&2Z=g^&kem42;630B^4?g! zA(o4MQyRB>kCcWNzU|rPZAXfCJ9eZree(Ag{9}jyvAlo0;2S^jNVW0_NRoUiLH(uE zS~z!|oS9viWXm7DKz2zq=z1MNwBJgi-QN8+nd5GPb|$1S_mXB>nvtTYEa2Z`7-3VV z>#3w}+HEKimP-;*-fo-D9uyIR&5tRgtJ!;NX+1*Q@f&|e<{+9Akr46)I=;flcaQgJ hTTVWaK~NsIK5Tt5l_%#<1%=2(5rmfC9MLvB^9R1?>3IME literal 0 HcmV?d00001 diff --git a/app/routers/__pycache__/emulated_session.cpython-314.pyc b/app/routers/__pycache__/emulated_session.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..10c79d9d3a5fb6fef71b1f36aa6e78530c6f32be GIT binary patch literal 1332 zcma)6&1(}u6rb7MZnkc^EsCZ#g{1LwwHiMv^&mpQVv8W{QuNYxS(2??OtN8S;zw18 zRq)`&g9j1+00sY;&d=DnTw=6(F$FF9%S?IEy!CN{kb z5+PslBVKY_X}t!eP8Nwpr-?u*P17k!NK>+qGx#k}D^oq9Cqovb5i-_(Ce#IWraunW zm6kF_qc5u`)%DiEdtcWp)-1K-ok$T&OOet*C&M0#9wKJ%7CK)yrJ#3WX4>*=!ev3< z)ZE<6UPZ9+y~|S zMsoj)yA3_lP%^t58(y7ICRDnT>tz3P5TeHO58B;5DqzWNx)4MiN!dCKM;-83lrBCc zqzHGzWR@;di*}JktW0SO-!Lp1C?Eq4Ak-zFl^qeHkz+})1xed>DivQKf;QU=R|Bv) z^07{S5rq!y;PIY^q6OC^tPB-g9kyWp|6@yc##MTQE*aRvJjMx^MJ>sa$J;zx29Id+ zrgGhT33#j+b1>*El$>4iYn399Ux%pyc6Pxpx`A4Bh2xb>Ibw%)%M~Ftm;&KVDMTbt z5M_=zj%Fhq(&+d1-0dhZI|z>Wd*-L$2*Y5Qb-NFL#NFhOC=Lx`uG#w6@+{HmfB7PfZ! z2mCjFfpB0U3L-XYDm!QMi3>aX4m&gNvv*T{y*)td{qoa&r1MoJ?ZsN6zDlA78(?Dw z(kL;iw3gA*OtL7f<2cN=N~ON8XM^nl-P2~X*vxEwuv=}|z6ppMAL=TOUSKhsvCCEI za7wH~u_{RB+|s^Nl4X8$98Q!Rc~IuN2_f^cDhgc#07ZL7yPl>HgD4TNb?&oTM|MP) zdI@aUq5eaxv1MNmOn?c?dXl*q$CbFob?Wi@w;h>l`RAJcH&9p8yn%eBlHd(+(MFuE z9C>~J%eng0CT$U+@IioYR_w`2-do8BEAT?d)>z zU0{t53zA^si!nqolBkJ^@Fejsux~yV)}&>W7(ZT!F`@Qdx)91i^?YTTXLgAR=Fx%hKm;*?j9}AxFu8dHx)`%4*Pb|JL7ty z%xZqZDY;C{Ii7REWxnDR3u`jHDS2(SXO;S9*aA&^lO|Rt{VwWCnJ(lVkKG&5|+t zDtS(^DC&Sc;7h^Qpqxz z3Z9iG`zjE!Z9AS<7LIUj+uyp@&AzoF1B~!CAirNC7s&1ShVw7|rV>h5V2fNQBWBb$ zY}<3@&`iv>=gKqH67uka(^aR`l(gCQY>{zMDtfM02JxmfKbft11=vb~&)`q7U^_yv z;%jhuJkVT?s^FGNwrx_j5#7R64zf)^SxZQyzfo9$zn92G!ZwGJH}oL-zuf}nNZ!J3 zU>$iHY zI9q0?YkUKa>Geio`#V>@xL8%Y{T7Z8p?^mcW11?iQi zkl@FMf!u{W{hT(h^?YGoHyi2x#qIq!wXOA|-=#V~IR4)8EBxcyhqY^a8>zv?w!wOS zDbaDo_#}Jd`LFhT{o+D=Y(X1aj*uPuZ?_FTATp8h7*0fF=pTpyY^v`>-P zYmvB97;0lXiaNaJ6^dm)USKY^H{tM83?DDQ{e&y*wV14OhDG7WP!OQg*8^ypI>WBZ z_MqJGX1TQE#{;dv=qgA#T0wk<2<^e6NaU?e1A_?tR%>5MKCzhWy_xL&CUamhGkhyE z+{helBu4;~q2#kS7c&QMWezqngNAWY| K2MNHr{L`N&7sLbr literal 0 HcmV?d00001 diff --git a/app/routers/alerts.py b/app/routers/alerts.py index df8baae..f0d2b75 100644 --- a/app/routers/alerts.py +++ b/app/routers/alerts.py @@ -1,5 +1,5 @@ from fastapi import APIRouter -from app.services import alertmanager +from app.services import alertmanager, cluster_inventory router = APIRouter() @@ -8,4 +8,9 @@ router = APIRouter() async def get_alerts(): alerts = await alertmanager.get_alerts() critical = sum(1 for a in alerts if a.get("severity") == "critical") - return {"alerts": alerts, "total": len(alerts), "critical": critical} + return { + "alerts": alerts, + "total": len(alerts), + "critical": critical, + "cluster": await cluster_inventory.get_cluster_inventory(), + } diff --git a/app/routers/network.py b/app/routers/network.py index e7793ee..192f74f 100644 --- a/app/routers/network.py +++ b/app/routers/network.py @@ -1,12 +1,9 @@ from fastapi import APIRouter -from app.services import prometheus +from app.services import cluster_inventory router = APIRouter() @router.get("/network/status") async def network_status(): - nfs = await prometheus.get_nf_status() - up = sum(1 for n in nfs if n["state"] == "up") - down = sum(1 for n in nfs if n["state"] == "down") - return {"nfs": nfs, "summary": {"up": up, "down": down, "total": len(nfs)}} + return await cluster_inventory.get_network_status() diff --git a/app/routers/query.py b/app/routers/query.py index fe88e8e..d022105 100644 --- a/app/routers/query.py +++ b/app/routers/query.py @@ -1,6 +1,6 @@ from fastapi import APIRouter from pydantic import BaseModel -from app.services import prometheus, alertmanager, ai +from app.services import cluster_inventory, alertmanager, ai router = APIRouter() @@ -18,7 +18,7 @@ async def query(req: QueryRequest): async def _gather(query_text: str): import asyncio - nfs_task = asyncio.create_task(prometheus.get_nf_status()) + nfs_task = asyncio.create_task(cluster_inventory.get_network_status()) alerts_task = asyncio.create_task(alertmanager.get_alerts()) - nfs, alerts = await asyncio.gather(nfs_task, alerts_task) - return {"nfs": nfs}, alerts + network_state, alerts = await asyncio.gather(nfs_task, alerts_task) + return network_state, alerts diff --git a/app/services/__pycache__/__init__.cpython-314.pyc b/app/services/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6bc54ed6281a89da5cb9d799a5b8ea7187c6d43b GIT binary patch literal 123 zcmdPq>P{wCAAftgHh(Vb_lhJP_LlF~@{~08COCqtLK)*P( ys4O!%wOBtsJ~J<~BtBlRpz;=nO>TZlX-=wL5i3w3$f#lv;}bI@BV!RWkOcrxm>6dO literal 0 HcmV?d00001 diff --git a/app/services/__pycache__/ai.cpython-314.pyc b/app/services/__pycache__/ai.cpython-314.pyc index 2d48bf4acab8836c41a3e027c785c0a90769c34d..3f849701cd8d3c303843ba7185dfca5292d99754 100644 GIT binary patch delta 9364 zcmb_i3s79wdA@h|?)we9ERO{i@m$0kBpyOCk^mtgFj`YqlQe0Aq-Zp9Cu#rx+}&k` z7?0D*9ri!xp67qg`Oo)1E+3v_fB(;{exqKeX5bOd{d(kX@fm$G`>$v6FBe!DXR?(U zH3rPi6v(I2yv3OYc^l0qIn%QlN5-2!>)~?p*v)qsAZPt6m|Uu&$rUqA-pi(Sq*s+oJ(qCx@AF02@{kwn;GVKw^Kr_ z^7JQV?UL3chEXvYj8E>9t=Fz7RU8GWldCzvFvC)Yu`#U)4C9LLukgvpn~D^Tx{`Iu zU`a}+fv~FVM0uMT$9F2)j9H9}){-nm7I|APmngHzydqVGg(~tRg=3w@Dxy(lWy3mc zOd8`#A8c_wp{agv~2}Uc3WIArL>D9Fu3C=Hbj4z641r5C<98WUytIzohy1PuQc5%;++> zGU!X@tw}`Cm4C9Cwvv4P|EFIA`3F66ctc;nvK8bHhC_WC9yXBS^+@nA_zg(1frMEv zpNlUb7z|&5B?|9QpVvL+;fvwTk1~;$muDBB25~nsurU(+`Cle~XUtAm)~DUnXD;e9 zXH|0*A$`g8uFE%!R#rK04H;?$Wi7dDD(IdY4jDEJ%FVx%GupJ7idk1kvq|7KJ-pk* zGTMrVVNIpq3Pb*rp~c2s)60>j8d|crYbA1^11!|x4<7^$JfeGDq+@FydA&kQUP-Xj zI%T~P0brO?8K8um*FvkfMbZtxwevrb1f6`jjpyj+iMcSw%CIi^&H@@ENSeyXaC9 z!E8yBuikYRQx(}#7)jw&e%WGSRpcdesiGyWgF3DQ8`nWYJIJ(F#>J9X%vmfQKT{WN z0SeZ7%HUEH|%X>n%HKzgFdxOjq6J@uC5PuR1-mJIMCzY z#`;=Qw6`-M4uJvRTpDL$JVL{+gXq&;*ZMSn^yubKJi0DAy20X1ifI~smv|Odm)e;e z&ll`LN|#;>_z&wzBHxr-HEC_E(+1_~xZP%nOH0Zd(sDDjOobx4TG%UHbf+Vjeh=16 z=a6R7ew+)=g)HWoZB1-zinw%_B-&~4YpaJdWsf@?NpuQeASyskLPd?w>8fj!;pQbtSG*BlXTqKRn}$T>T1?I+6C)LJNI9APhB~9_4qrz3%#Mv{SSE$)CQ4z z&_TCfdey#tATToSt+S7K{XWk?z#cg2vEx?T{YS^2@!E&RdAr9waMbP|8iH!ipuN4> z-tU28kKGp>9Ubuw+egQT{UwtL>cYaEq8&g7@Y@R}rH;Zvev0bH9GH^ojW9>`A*#qQ z$B&PC!jkPBK{>qMp8l%6urRDVI_?i(n@O&)&|U!6MUdw`-oT`y2&NivkA|5bDg=l9 z_VighAM|>Gi8?!sJMQ&N>g@JGWc;Xy_te>gpxDS$DF>_{NuY(o9~>KV^UsEru-bsz zJK!Oql{h! zb%(hDkl8TzjGOm{Ww1FOKM$7@7K&RMmUxH4iU`ADC9(;^=ikg9rAx^PHA*9VA>NI--h+?REU#MiCQ0x>e zolD9tQlFMFg|ZN;KM<-sAXFR_EImugL(59t3))j!!7{X@9R7tuI~|OcSPL&z&IMnu zn=6_h5G-{|$}K-Jr<`nDE-HJY;_^WS|!aP=(i z>^VDmnxDx#H+gyzDo^ebk`65wm%njf>f(WU&w_KQxFKXcyja{Y%fH`zt@(QOyH5)H zpIU5u>b!bJGPC)DdPWVECtC#T;pIf@xtcRIGtIM&p+pBvt~I@|_tf6ygyb3X%&{9O z4k5X4E@v(A|w_nQZWLr)1+hgTT3kIh=n%Kb*lt0{9W)uF7) zn^{|htgZ7Cf_>YHlr=T5zmrOJ`UgtJn0)HQve9x*dqz9+skyE<4qiMseV-Q;as7JHCM^uQ=soJ{g#JFV9iJ)sTWj3WERtB@kK(Cbpv~zSmOR9@=O5)WX zbW0=0i>$1T{3bJ-MW#rKRZmRz#N>6f5C?iDle!K8`<`Nlgdc{rwo^yNAzj6P{vZ^u zaazbP?Hkx}qRFZ>VkdqSNf1d6Y0q+vqU9e14a?j<{~6wg<#_c!k^-hpDgntx{E+)d z0*KfL<<$I!ls+n;RHOYlWMUc(N_?z~buw$#%YqO6oXiX!w{=Rodg5^#$yIO>t+z=u zOq^i6qt*ih{uoU0kpys+Ee*>Me*w50?XVMn1{DVE>rK2$U- zj5I_2^W+VuxfLm6G@ssc!;mTH?X!~E%5T)XS~J&3L1X_yf2e4CD0_#nVdvHI8``FS zOWyF%&%q$iZ`sODD%w%M*V)PStZi~Xl=1oVJLFp%-Zq#L&n2BnqL^#SnchPJ**QIecE@znErn^>oP2Kg znca6HTXIhZTXT<*TGS_1D@sPCN5DP(%*?YhPYag9CFRCtPW9Qn&+R?EWrn{n@#4hn zvH9FkW{r?hyI>MbTb8)`TVx?SXenh)L#!Wtuk+iryVQs(yVPVyPOrovA@9`Wv7ftq zHAgL#KLstoViNL8rjX%Dl^s`E?;mL6p|{N>EeTv}?Jft@G?EXC4;KPRqU407+uNE&*j(Qn4d)ZP(M(=0PGnyuZxnxdlh~{C z<9qR6g|W}WkN+By*T_Ukt7I~b{Hny3o(g^K1%8j;5B92k%05G^)UN1#eg-Sj;(OgSz;(_X3r8=S*Hj&valrR;Ty4j4-z;Ph6g8} zaVYq&U>@&6Sc~3+AMnEi#w3sCH^rc^zAq9I>+=tc14hw1qm08O6zZ_lJvJ1U`p1UE zi$ftw?cW&ekARe439Uq9@JQqub>kFB=q=z#X?cCwTegJ(Vds%hxmzgf7fb_7++YM& zgv8=YqYITngDX_BUno8xm<}#+J>>oJ;7$>D&SuOiLZ*Tx&atd9yij|p_H=hhlO}L! z5APb-g#9daCax738Z)?S^=6>oNlZgpDF+J1%T~O@*0pz-e+9Xux3%qWAsf4`8oaGQ z&cpqLs3-#S@L#q93**o(=(1wAgU_9r_wSF}wgNAH#2 z6A1dyROts->uJy6qzuM`8)gssA%+R+((6H=5VXo5x_+w-`5YV*_L3npoByVdst(!e zP2uU3-^4Z%{jmn%-+-#HH0TrWf*<`sQSwFg;liFss62g}e4)Co^t^3$K-kz6N@*67 zcL}DJC2n_Aoo*8A+CnMqLh_S>sbh)TN2E2Ijcdi8)YN2=j+`dl^Ed%&8L)n6__A>U zUi=wA&=}{~PWBF^XLD9)dcIP87v(h;cYyZDyA*!+PBWyaHopdJxTpXPgH-_n#5mb_ zVatB31gu(#6HIz*CB_0Dx5~j3mqIvJMomnL$e_0Y7O80o=56~FweunfO@Nt4r3ebb zwpiq&q@McK{%s}o!yYiSeEha6*d+-FjU?oETxU8^3Efi!)oEsVjTv=IV_7zsPUn4bQc$KW>y4+AzIg0pFZg`5mE*4k3M?pm7SElkPn6RwS`|AZnd!EQ;MR_+TXMm5>VwmP~P*&~l>8 zwTI;+V?OslK(r|S5iN?=cc9jfqKhXjXLIi$IombU_*aKkFncMRCT~G_8s5YMmq7FA zgY(h@4>RO{b5cn~Ery~Eib2p9;9uc6vQ|=Bn=GT@Wb&7F*>Z{&nOL|@j@E-XmQUhT zx+LgmWiw8dixqPYbr7exeeU!2_R{UEH!XVFd#89jFA*}OKF_2!=zSUjCJ<omldkdPCPep za`@9zGdW9&jAf(woa&5fF{O2$|JKBtjS~yQSNWytJ)xA=kg@fq@wi|-ex3i(#P=rz z*Ab!L7kcveN!hX~ZKi3aS}%P_*hy!WE$*)QB^AQFJEx7hqf(!{SFU_hlgM7g-S()|bYtN6n)( z@;HS6^g|ehMpP&OSr09C_gwFqRo_V4D5Mt6?U)O^apK~M1^HDjRN5qzG+%Y!NNx#r zN0Dh`F*}(Y+b(K@*$0k#P@~n^`G_dev&KycD+WCQ*g(H1jt8SY3sqN0s1(#e24L%v zh#UBP8sm=*o0s(oFN~ZTS)_c1tcODSL({vKb$PST%#H|}GJz|Lg4I^?A!i}4)a=^z z0j@BM4*CELA30h-a*Xmof>9nwFvWiT~|JZ;UHzqLy*QI`H+Jc=>}jh4ycClH4jN_)Bi|8T_dL~G38ksKcS%Vcz*o% zkvuMsgjvP{{TlF12U;>Z}tQJfw81fV-}$gUW_p@#27F{IG_ZB?h_un7>cN~4-ngy z!$lWyBxDl2myMW5f8Y@{T7snlke`t6ZP~P4lw;-EFVP3E6>2ka!>NF4aqhlI_&efO zKwzV!K2@^>EvYD)SkhU4cXMQz!WqfUMs*VjRQydyX57my802LdPfTbSee-C(Nz9P58dQg+tCSY12kCG77n%bKes z>`hYF(p&r!sGii7x_u+1Xz}oF`0s_EVQ;yQA0P7t!gBZy;Bk*NJ6QfUdAFsg>_4Cu zLQ6Oqk6Xt7Cz4+yQ6W(x5xK(LZ%F3u)Gf62f8qOmB)>(ng5)j`KlY1-`b9H09V?Oi z{&HaVU$7=?Skc!vI6lzVN54_GCd?ykNs8Dy1+Q!txinFvSpEhhl$8}&X z83`J8VP*Z;_+W6eQB6~1rPZdtbFO~zIt%eE5Bie<+k#)%0@oDDCLZDqm4RvbAov1R8yiN)a?XynuTS6P&&#u(}I3LOBbe{fjU5jrVLEaxpHLHK!^V5 zo6&dgyW6|V`Mz_mKK`}vhkJ#b#W^-J!{?5XzYqVzdoZV1_}RhgDcQq(#Z#iNU9R*B z{__mskUm4n6OBAMUci&*J-iMk6_;8mGZrzy!^#556c%8W*9k4_toc${%4NNBLabv6 zv8{lyeQV&(jTIibL)=uzn2D9Mab;ZobgPg673yX7>~`w740c0_#nd2#!a805*e5Fv0g4{Pub6;q`{fs$Dq!GUj3-lZWkK zA@c@ly*zHHU|~ZpYpY^x+!!`?39xy4sn~6TR}3Y>SK%)Pzd~QLfZyQDpEoKmA2uw2 zxUrGX*ve<@pc#K^2A#&mHandIJzIi}B2<_b;B1c={hzTq8y_hTaCIG4ns%Qf-80=c zo%gW5{7h*$EH>gk(Br{#sKDR2Je-yD<)@_BkPaW29ug|24x2?`Z8bL5Y6yG?I(4sd zPilB*O!E`eAyB>UMEK|5*d$Aux1M!vIS@G@e5+~NwdJA~nEI3D&qCD~8frPbV=I>f zI2KIW)_t))E%4=>|FusmSnTCOU}}-QMiACVb=tlpX$?*oPgUMh4 z{3hQd=F0H<{7p_NEM^h4jELH00KxC)R4vb<>Swjv?Dq$Vs+Za!RYOR9D+|d%sZJO- zgc%tDJ-e2X$%YLODRD|lp9K#3osxQ z={}032+Lu!bAiCe^jrJrAo+u>Y(eQ#A7rzbnZ+Y>Ux3RCt#GE;y3L;TCFN2t93wZY zT+J$X1PzzoXzWm4ZRw>iGlsgc19d|V%X}Xfg00SA%5N7!1*j!_rWMYx5l1f!%b}v| zo>*4Qd!2XYD|NkT<~0pFTF^i45ZS<={#(#%7LB51TwS@Xng1nj>OLCNuBg^gkkMi*Qxi76e^HsB+JW z0K{F@-gYWRU1wYXmUX-wJQaDm>ZIuz(<{M?@1v@+TbxDktn1S7%l7wUSz1^k)b~Gz zq^g|=om4~Y;G?$mp6%*+ss8EfPi}i=+jLjYMQsqP5>)SYyWjfm!|qjs@!_#pvwLt) zba1bGV4}i3GPYZHx5S244!DP7?)aXlTT8^^!=q8fy=P31!@y;I<=Oy_@Gr61oigZ& z(b0%@Q_46T)8moYU^K;G`m$B~jT+4l#c}Kg+_ShgL_=say_6amACJa%rKFK)EG1tz zHXPGvxjxf8j;6%8o)RJ{c@XzICEpOyVku>0I2P4Y^6=nTOgCXK6cSttKIy#;W08hn zWSrK=1$eo9A8f2B&%+I#Ue+;P*OByhCY@d9jGJMiqC7x5G~LoW-O!s1hLg@M=Zsrl zGurO9-eFA^`Onq0CjD0?uk24c2hJHIZyRjzhWkaYk6Eu0P={9OMfkZRMeA0|rxlgZ zQM6u?Py1Cw6r$^dt*~ujbAS#mqcC}-FbIdG8A={SX~EJ28x^RbQ;#zG=VuH*gp z#Hl|nyh*UAXx%y84ABmER0fpXpc-;OtGC3P*GOB957wDcHAGYmCBxV4vwS^US=iJ^ za?PJ`Hcboou+Ctn3`waV4N<$1h6XrW>2wm=#Cc>Zf_oaa0tw|b#401ed*xdT|MjS`Fyc;Nc# za1ME$1Z~<|hi}t%!<*HG(BNCCUPmqV!VNy({upL`Mr3L9#yB@dt%EAb{8CB+%~I3$ zQGrIFBlwJbBHXcY7nc%aLt28`o6xC6Mxt6=*KVTbHzMdpgrQlJOj~4#>IiU|64P&l zmPIGPyx8M@&0cz6@WH10n;u&9SajOem@HlTg=o8_BlTHiDzJPt>-<5~V_jPu){%mefA_N-O**P`5vd*$I=I=rSDJZPs>E)gFRL zf&qe?3HBmPNJJgl!wDI41KNJ7;oBsqH;8Wq-xR(dYTD7$4rtdC+(1BU=ewY*C74Z9 zGQIw8sv?L_jV$RA;42%JpnrCj1tbqUBBy=xLTZz7%juOZx)lrc1qkGqt;8CH!@FiE zd14Juj&<`wIKA4XZbTA96!@gMS`gutpw}dY$brf|GRD+}d08IJ!5JaKn;xeNt%ZVI z^e*U8g5sR{@@`I>DJ*4}{` z7)R6YM)(?K4iFrKfu>D0nuqel$qOn1a?;(~m7th-=D|b9+-o|wbhzJ|JmiL>O=Utk zoM|d8A{|eVG0bc6*QN(n3}7*5e1}h*qwcI^q=dbkl=B3Gy3o|V0VkHV3Acc9d3Ra@ zl$4l=YOARp_nv9%3GnY?)MGRTN zQCT2{WjNNdw~RLC^TIQ@VujCgh#DLqI0$1aqO;WM?fQHkQrm|RzC)SWo~D(-$~kIj zRF$AWC_|YE1&3^Q!`D~-u#*e9lx5O7wnQ7B6Wj-|ASETn(}t|m330nki+3-+Y6Se$ zpN6|yn^*j}`?2mPi;ng^-E+>r>i*(WUrH8tCmlWK0(|8{7&x;~8Uf@&t2Elmd){D@LH{W}xGBfI)BAdXX1^ElS?jXlG_*tr89>1>9#JKc4Ph<`85 zNV1|{d{1H4W{h92;Qv6}YhSI%Kd~X4u&*}Dr}J!x7`<+QT%}bwI(4G!3Q0}L+So`G zKIuLrrGx<7+w+O=INa0QtUiOE(>mdIyI~xDJSoY=uz!gRR41*3^(IY zo5N3v!kH;?Tcc<=iWMCbE#T-`412Cw>_3L@6ShEPd^oT>8jEU?cvSlZjiZihW258o zl!~{AXk@evclH@LbxrM(6IkmrYxG&ZYVBFdJx4%<&?pwu((|L-^HA1TTFP6$K<}>- zyh!j8!5Qf4bK~Up_1!8s3Q~st{#|2({r&tJqiqItN4+mh?RY%bO*vHaG&#~=$E!cn zE+eRcu^q+cvzSGdXL>gr+)>kje*#|U67WBT3qcI@FD$YlxG|KuP>xc0p&X6;LSd<} OS@^KoCiDp0;QtMP4`CDl diff --git a/app/services/__pycache__/alertmanager.cpython-314.pyc b/app/services/__pycache__/alertmanager.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8294fd05847c0d86da476e5f432e87090ec3ad09 GIT binary patch literal 3045 zcmai0OKcm*8J^+3_*haDsTU+!q8?Df3Ke77v12KALCd5pSW@W~B{!&IVoR>%WvL~x zyObpX1&B5WI)_F=3Wx=YR)+wU(L;Rbp@$w~0~K;qOtoSs30fehpcg4}YqY)epCuVF zj)e}1fBu>OIseRlGs7OY6G59Q{X>17N9YSWs5hp@oGbxTKoT<1T_oWtY+~qbQ?{Fo z$@J1WW{SPZnOrY=n~8~a^V7Z$N$f>j9h-v0wIhjdM;WhamjvML7VnVo017%bTEZDc zH&#2cQDNb#BQYMnCnKj@jWczMUYfmh^AoslG+enSIoSwK@c{h8YcAo zQdVD)O@q4bC*5)`7cvxmRZS{JNNz}aB)3k)L_*fItSOsHBB8s$g4Wsa11X^As0;g! zDf!i))t$gJN>Wm9O~CvNA@*by675I5VAX|A6OGXnUub>kE+^s3wS|x41e3SrnS||P zO>YuQwlsKK!v$uVbD|s+BgVbp1cr-n4ysLrogKAFti(yYB!p~|{cb`RJ3Zu8}42Fui;%p4;UTjix+%Eaf z$ipZ3>pmp!89EKQO76HrYKc3g*0?Lg*YTkj-TDhc4%CC<>;Mu_1Wzwm^nykC02;2I z4z_-h1^Be;Yh8vt>_)1s7oCG|hW^05XPh~QR2)EyUt<%GQgF(-D zMd^N>QE{+sV@$FzAfV9?A5cI^1t=B-vP%IuX{xJA!20fuz>=O_2{Z!N2;hZasw+x1 zZ_aLbLUK+Gt&Ub~e2uMxq>udK)bf3B2bBy}}sf+|78u4;xUgLA@K26jVPRdm%{ zC&GPM*Hmqp*t8|WX<4ufiiEponz=RN3LEQMa>4>%!YnH$;S$%wa}nazWhg@ekT3y$ z&AdU_v;mPEk+q~^1=Z;hB&}eR_%=8@0H~J$t|6p*N!3!oB&^U-l3Y&FQjh{)#e9Nw z!szmSBGlY~3I$n%U|FX}SZ}4Io01M1QlP$|D0%=!2HomFL(?h#5+_Zo>PEi+6Ba1{ z1R(QE0N5jz=X{^}I}ZJWPyK_tw-5ax`2X1-dUUDM*1etHN`Da9xx72F-*@GsH$Gn2 zpHJ+|_x7)=o9S{}`p}y$hAX1`Q0)0c>^Tqv6`y~5Xlv;GSBn$R1=R8yxO(P6twV<` z<4;@0%Pm(AJy#2p6?ao1`l;Jn#Fb{>cI#H_=Gu<0+&oz1D{bxD%9c`^*&QvnjTBv# zrq=g19&D6`ch<{I=ZkE`cWyhh71{~z{_w-Z!^9un{`h+NyWcPSVnyLIUsvhI?xhdM z9*#Xqf7D+dj+A}TqVTuK^nthU&>MQ{4LuSLyy1N*UgRov&wK6%?uzIuu@$kc#Q(u*ocm4|r4%cxL@-!dyAkkNt`RY8ZmUz?8 znd+)x;RpWEf1lZr*!m1%&R1z_e%Eot0QaA-c+_<5-^QZ9Jy-W zWq|u#f0XAQyKxlRAA1HVKk9<%$5(NbVIN;(fq!D_0J|rhIC_bF(lyZ!gFk_I0PP_R z278Rf^L#Ym*b{NI$+PEnQoh4RX(x5D*JA^#y#S54H{gz5VD?_KW=1jPFR&o}#+$+p z>C#hvaudk^ds9fZ(`#E|8hG}U4*}E#Ih|g-rw3Z^qC)`)00Lu}dJyaMimkpstn+L= zEG+UW6;MK77aaB+Q}T<6Biem1V0FLuTiqZ1mQ1G~NV0Za0E-ZP3RtV6dC^tD^$*v! zMQtJ)NLl(6CXIhc%10GXMI5O!|z&IDm@NMRszGf)^V54fR2 z@w&mE1q4>z09utbQy+y%gDz}nXebb<)+|UBDY0rc4om@6Jig+>=K0M#n^UFRrL|Ie z-xWM`g+6hG%C7T;8=v~sqEK=9e*V3Rzw_PMV-B^X@iC7?k5$H?lL(1KDw~9-ULW`) zP5`r!WL8^Jmv#E8AR?WpKNhP9$?LGCYEB;nxrIemP7jggv5HpqH&?Q$d`7vZ-vT9C yePb91d^#Ba6?J}r#Lv+ypQG>m-QIC%?|o|T-C@f1!2-{qwk!1UiX}HzsK+Uj<%akdID+pmH!SZ%Lw@`KFDd4RABC@8A9TOBT;gU za7;hbr-&*lv8?D@Qt{lpnY1zhAK)I5Z z?OYXZA!y#c1o;6)=w|B7juuCB-BjbXokMQA%01S!=qH5QEXcd(s zyiZidBBP?Bdq`9~GbAeaK|#6C-76{&_wN@~EwmrovLCA90b8}*hYs#Ngu^wol&zve zpBd`)!*?Y3!^FZ*36J-xpb#e>a!k?o1|cI*S5BU0PB0w9DL5skXu^H7zgIvvQF|gR zL_@xbfT$Fryr|{_(HI}PPGDE+(J(*ZiwZb@jacoAM4E&Ee=6t?2u*(J$h^VOsX!yx9P6R>eU6oOFbTg#`khcC+xHmei0@p zmy|*H?q^Le9P4U$-q&z;XG8mo+xY@`0go#b6$hV0Rhjlt_l+mODCcMxM?)Nd$EykfEvm@7oZR)%bUusmRu>Bt9fj; z@27(w45nK4-QLubG7Zdc>Y48P58DObln;vW-Z|62=bigfww@11XPa3 zwP@e@;b{E8oUP{z>*gsoZLlmFHm4oM-x;{0BDRWqYGN;+Ql$;n3!UdX6I+%HRksaQ zqN5mE?d5mW#Av;L*G9~4=Duj_5r9X?jBalQ`>$#sA1f;MI@vX%h%8507&o?&C^j)p za$U(6K^Go1v&fDF{ZSsp#l`TTyNM)v^mM2=QL-Hy96(T}wlnHGNRmfr##cbSAAZ?{ z6Tgm10a>>Ojx<7_$33hRsi7VYIJ&AyzWmyiGwj4|cRVajGAy+k$Cf2!u zFk-7L965prym!);Pk1>sr;+=EFu@t+70Rx4LaEK}J6ovE-EUS9v(02pJ1BwmshW6{ zvcRp+VP)N`V!lRQ{JARwm9#$ng;u6oqqdNB*Cp5?%W2WvvgCnpuFg?;`Nyg@9II(0 z5$!OtDoy#-bS`Je=W8WBOz#_nOoANo7l#Yw8IA^bpmb!?FiH+hGN;JgIReVLoUHAA zkZB?jea>8G*31R!{h@QM>3xuaof$j^;ACS=u@g;-6FB+Tc{y1x?}{_DJ)2efun!M_ zque#TtQgpm;U-hW=+Wn$%_z^-!N|KEgF-n-c8*-<=VK=Cj5(4f&mA!jqxzxAW6suE zJlY|&>gy7x_B?WC&h(dC9cT2IIrB#Elvld}w`FCyt&b4SvcW8tRc*ytJr>U9u`~ia z9Y%{p({NR$kQaJ*EW^MLQj)c)%%k%9IETl;6|BzaG081WS&pd$IT-l1#1o)vb)521 zyW-ek;2|k((3h3pBj=pTqeacFAw3HBAM$~QOdY@n+Q$f#ao@?*8iQn)GRU|J2SrUF zBe zTzv#U2te$BsDX@+21X=cq4AB3@Bu*(HJLYvMm`V;3qb&(!GOT;fW3<JZ?)L6oMgmN5~)Gcj7x( zIGHHLJ)#=0HXtfdErfCBqJg5V!CvpdE^rwHz+dd~aCqEhg;S7#I2s6ZZcbg6LbLgCgQI+k4R3$8VTT!V{+bkM=Mg0W@$rb~@)HYT^t*?VrX z^Y)%4V^6$ixv>1wu{Vz;Uz{r(xH&RkIIv_Hi1&S2Qi@F#iOH+yE}xreo!y(NY+WjB zJFlBkE^n@%QT?<2C;OH*w=ZmNpNf6=>g!TufTgRtN zpKohT*`K@KkZk^_^~2Vit*MTITW97wo_(uhx_SE4rH<*2>kU(yIs0<}axb-AY)c$S zIqH`jgL96-A2$DJ_do5vQSed8hb1@bZq=mP52m&cPHC4drHQtbr550^)^x#m-uR`p zD8XLUUe+dyX6&hw9ck;Pcbc#6zOs9!;729bOJ?hC)TCUyQ)QiLYiXhnS`#f-+b*{y zdsAiGXC{Ai?)teKovEe+^JNFp)`}#1P5Z8Prs9S@RrO>VXHHv-6SavVXnRln-nnb9 zzWeHI^yi0v_Drh2J5{w0=RBCU*1Xp{)A|pGRtgO!^Vdp~PJhQlirQCp5S@LxC9N$= zXs#MA8SK z;QYUA+*bqHukBW(wG7gF70@-V1VUN>;mp0QP}s)qD>Q;!F&?wag zJi;3)<^QWkHS+>`8ZU|lj8k|~(FKIF7OZ;#RL~9xgg`6xNvLXiq)YM~%r3a-`o1PL z=#l_QI#w{7d7frB!!($4oJq&ou zXTc3nd(=a@^JRu|0M%sE3q5_%Q{&O(?WvFrqDR>_DaTC)00?nPcu)Q@S!2qna=;y^ zQxBUTYU$$Ytp{$Qm? zm3=2Tpx`o^U={3qJtWJ?s_76PJuh|z)n1R0Aeh{6qmhMGFoWhj_|nT9NW}^ zkDWk7O)T_sC_EVwwOKpciuH;nhG+)RipHHH6A^%&Aw@>*drLq)q2 z-^2rd5Pk%RixpMl;mH6`Aq?+^Dv^CD91OV({EOHLjXv+iYNLz{C>qm8#(n<41fVwo zTLK{&C-EG1)?_+MQBG;$(pNYR%>v4CU7Zxy$%b|uc{aidZ#7mz!TvOT=HPKn7HPlq4wf5;gES0VrElF3ROl9PXVB`##mAO|7g-6%~ zMH55a3K)3o~K_@hdLNyQ!I%$~3J4g}*E11nFX*SE!N@=yo zigH4YL03Bm1)mbu%REfChn2%9h%|YW%?w8%TCd%}nOURVW$VDL>O6%pyLReTfRUsZIJjrC02*l+e&?SFA zX>*ivM7BDI6*(3J!vjo4%~U<0W>E9Ny#p0&my#caNB(&v<3I#Fy}GKn6gE5o4`MZS z!ycw);yBO=|(C! zVGU2g!yt7wBqiX3XGtIXu`~h%miQ86;G+aiN2Nux5-Ux-bz6aRM-j@!+)s zoODDnoMk{y7VOY#1{kqEaWFOu9auE)&%h-Gy*??2xm3eh(K)&+A1IK8QJw>0r62?t z#f2*gl}E`jb?{111$iz~RwJ%pWx+z>C=Zjp&``lOhJxP%!$l={_AZsA($G_{ zga!}v9(8Tueu5ABA?_fVFv(B%ha+d`EYuu6a2-+P^+lsT|A{qR2mygtU|#xZf=~c1 zwAkW()J!cGlq3qicVJ4FF0Q)t%Eecb$7Z=yaod#fbISP;kqN!HR#;x*`yO=#+0MQ!@<#rTGBS6rk9*U|vvW9m z{RXB2(BAr#>s-icJi;E`EAjTQRqmf7J1`5#PBtcf^a${JM*2WPY!a_W)X=9;9?dBL zUaRxKYfVlciO5cjK@xE@FujqB+L zsSK?#bGTw^=Jr7$Yvcf8tVhZmlVjBEgdCHN8bA+vO$p|dzC|fn4?oN&0qE-rYS%;s zNI0@X214^Epe$~|5K|}+oec9YQv@0l@?hsH&{e=dHnnNSmdlRm$C9n@8)l;ORm~Tk z{?5~HJMP~@R(j1+#e831B{|;kOw<3{9!Fq$X^;H3p!mS<_}8KKZEVS|-Q0G$d74We zeSiCG(R@wog_GYo`F8Vt{tf8Fp9KP-JT@`mNEgFa8 zQJX;c15Z)%Jz|1)!WZHDum@^5$!Wkdpg0{ufUyZtIUWePbfP&M)RW?%IvPfxAyHkn ziFy{2k3p?GE^~Et0&4wiJ=Qu@QcYQ3qsZGa%%%(+EF$FExB-E;cG%R1vc zvQWy{lG3%r_bwaE^Y~d(=C+ifExr#?ym?AUYi-l}5;cj@2|Uo*ET@;6I)^T7WR|6PALCk ziwfu(&Hv~40BY(3Xe1PZ43XCdpva^YS|8>Jya(-BN5INB?hqh}mKDrqoHU!=Mk^n5 z1VFn<#>xXaAB|OhK%9d+0B-<3fxlt&0wY8Kh2$Nv#tG<+p@#LS^7hQ49#5`b0x^TX zS|G2lO91p)PMPcZgEnQ?1x-{NtO|6lY+XR}=3hC4>#}`x8>|_EA99~2m8`KV;l6L+ zZzLT$Fr7h2=;#?Fk)zUvZh{oC@4(UyBvnYB1k$?-*8xmX?36)FltHM>QtYIG{{R7W z5+ua_8Vr@75_Rz<$u*4zLwLzTp_0I(z*0W>M<@V7GCD2{oF4%EZBH3%LD8KGa?xx-s_Jp-)l>A} zKN|z7M(5sst?1pN8O`jbRCULSL0wq%2c_0#huE*J=w2~#)JcwhDKR?YgOpi2@e*EC z`h4(DE#JhbsPvy0<-z-dxj88!wFY6m&AT_hxAP7vgp`# zJ6!FDaBAc8eE56UZiHW*boF5JeWZs^y5VNlee`wP!k2?ZMs3IgBqF>63H{FWROyoF zW)WcXMkJJ%&}PzqYPkwTjn_L8_Itge-s>HW!GHGfFL#KOBuifnB~cI8h>iYmXf!y+ zqjnRGSRvo)(JN011o%N5SBs<$NduA>keoy^iR2X|-$jDbO&tj;Gm-+(J~B5u5^CN}uvlIbfo{MGQQ68MYMSHlckM|^dFfsoZ#1>i$}RRGS-*R^1{=uZ9* D#=>_% literal 0 HcmV?d00001 diff --git a/app/services/__pycache__/log_analyzer.cpython-314.pyc b/app/services/__pycache__/log_analyzer.cpython-314.pyc index e957cafb3b386f9a326da4127c985a99b49448ad..a5ac56281f2f47a413a609a9fef30843cbaefff6 100644 GIT binary patch delta 3578 zcmbtXYj6|S6~4QYtd}jzl5NSd9;=5R!nPzokO33G28_(Z<6NvI&1f*PEkM-BJ1Yrd zCW$@iG(-BMVRADqNtr1>nidENP}&Y@$q-1wxM@1$2N`>W(=zRkPKF;fkAZQUA3ay{ z+QNnONAGC9y?f3*=YHp$yJuJLyv^MC8KbMy6=@KR=*heBFV;@kaRC!|cvRIub)?~#MMkPX4 zgcY~|D}Bh%3L0KXKJnVi!`g6RTINS-wi9x5l!fiC@>+<1>xkHv_AG2+a&zh0&+e!} zNR8}BC=#?np&to4z6eCwF?g8nmJ3quM0^ps=&{yf6_)uKIa(Q`P#keC8p6^ZH;HnU zRmCkpg$_OHbmfS_KKi`K^brr_!Qs^JVz7G8-D+67!;|9#sVRw2L3YqMNq3cc)F2pn zBlznBe+qm;tf~_TNU$VSkIer315y4ya^j|DbZKkOlqKh z6KGfDC7~h`fFzhnq*}YH;%U@znY0HkU-JS&i$t3d!loX;l>?hihzC#Pld95DACh9# zqn5>$k~_iD`>aSS_zG;Pw=BDc@K{`gDS08P^wcUwpKf8n-X+flyb6j?_e2q8&ylphw1E6cszQmV+i+*rn=k2&C-$-!q4&J&JI@+Kv z+Z8Rm1J`5U6V4pe7dUm~TVj11+s5Wgz_)(sp6vFT!z)c1mek@@jGY$w!a1ckUWpT#aO3v5)$S2DbMu%cz>Tom>JrEN!8bKUB97`RH zrIVSGfuVFV6%+g7i6gN@YFHd0&)1ub8D%UH?H`H_c4icb;lWr^{1rJ=e@MpGbL2v0 zIr)1-&~cW{s16P%Q;F!|ShpxZ5NBDDw*8Epnvj66Y19tq?|~U)-@qrwgp;4#7`~V~ zeQd_nH05fV-WXnZKtBmyD>E_2REhl!gTqG?4dM`pMEaQ{egu$F4T$knd>}e>teA@@ zxpYE|MF$R2cE<`fAB+urpF5_9U^vPpo;bkK&5s$l{c$l_pQ`6TmEgp5g4+i|A-Ru> zCI-29A}PcMQXo_Al0(*2UK@~!bJ)n(p8GY{=Q@?qoxV(OT}IyZwK7wxS3ZGM;wM=>d$Yj7y?gR?R zKLe$fdm#9RYMOy2JsOWSNYTYi%0r~0F|>KkXqhoqPZ_HxYSzrubWhfFPuKKJ8=oKR zx?^(Bl(@&X&v^qE?C0!PRO2JF9i0zl%&HLc7%`p@_~6DmPnyW9{S$`9<4SU_@m2Oj0jX{J+F`c-T>XxE(#c;*UmN*oWZX1?dv9$F zPdoW(Gk?5`$eU|quyHIJC>jAzcXdcqw9$#~iLkw4pJ zx+0TZk?F3z6QRC|zWwtu#(9vrqc@*?dA>|rse6R9MaB1&$Y6WmLI(43#hkY6MC?@W zjMjBa>ze&eHZ0J2vw35d&X>&{dZUK=)8zb_=Q(0;`BpcpDmig zM|aDR!E>Lb30ini%9I4}tB~mhX5l_&k)eL!z7?6fnT6yJp(d|&Si)`04J#9N$#2*= zI)9D{SIKW|Xs?Ew&pdVE+JeuETVQ@u#)N(HoAQoAxEU{N*TdzHtBinOu-V0b^0uqs z?#n=M>pHkj1+?Kt`BYnn3~p|1u!Mt#ugqGQe^td$tW{EM)P&oJJ-CgzO|Cf|Oh0)& z_-?^%U?$Fy>Q+wi5T^Z?ceFlcv~yHCOW_vTwYHtPefiC``x)kA@@U;9z%y;LR5q(x<@7h>zfs ogbsl=Wjjhnwf^w%V0tLFLHzgSTcIFRO#jL~7G*4B$^Mc14Hq)$ delta 2867 zcmaJ@du&_P89(RR*YVqq$(h~Rx_1S@DnP;dmv1}%8p$j*pvLAJ1yv4Bn3G~mIA zoeLz(9{eN3fagPJ~;CNOUQz{{n5r zD%ammXdaKStQ3k{(@+d=PBa`{OWQUDT7=O-+W|}r_x!pECXB`rGNA_4 zffTzOE9U^H)VKiI9<5|k9Ey`Vq5lx{MTC@ED8i-GDQ=mcf^wslajhQB1z&0;28|u9 z)hxwBJzDT|nbc?sV(sItRa1x7Y*Isv)M!P-B)4h9)Sku_+lspM#7rH;0!vw`i`b}# z*l8VcP&aXsTK0TXy{`tSp8(q4Dp9<05=4nFc(z|%Xkc@pu%GzSUD9a<_VJ-9gq_(EMG6c|n!HL4Lb z`c!2*sR0Up84C7Q73?b)99uZJQReju>Zxx_Ej;U6`2Tl7?$jt#pGJyUWKgZr%HD;t z4O_MKVph_iosu<)wvLNv3-sgLCkZZd7@nOk$?PE#&s<&~1df7uu?^y#s)z@7(=Vqr z&LCt6gbWZ54Uo8;(fl+}z1Kh$kwgbd>8YOv$~OzHDZg3N4jB$Zh9C_RGt8;f&(kO& z8lwIkaFD0$oH;BJB5~5VAz6qggQTI2%k>NWLbV5y+p7CXn?@9&lCC7IXBl4~Ac^WJ z8!q2k5DjWF!{M5$R3z`vGLR-8P=oiXf&ICqRu?YYk?#7E(7~TY{!rA@`n!eEn#xS2 zL8&LrG)!7(18J4}v^?W?^4|jX$2Jq*#a~xD*(MKYgH&YyHiY4^H{OxbD>ivh8`z33 zY5Vf53#9pAm?W#W_E4lGlwWW+P>s+C=p3n*WTU2sg;8&%kdmN8Qo40%s4b3a(2#yC zQ^r6^h=A&fNipWRUpMx3@c2Lzr9KY`gODJPw(Ih^wVnCfza0`TTeCAcb?&ijAv>K@ zFPjy0_OaaD(Ojv>R@!}n`ZD`(`y&g+!jbl=*(VC^D*R+1mH$d#HcqPfx%^~yX5Jy? zi&Cke=CYGVvxjGL^CBq~q=)BSl2UwF$`+=i$)mZ+N2UA`X}(38oGr{{^M#zsXQ`z^ zAzzs8m-2<8lAD~9@^eLLW_G&Rp}qu*%p1z=SnnNg<5!u`nP4AxMDZ|7c7{zwJp?@I z@nLqLvyGkTtne%~(!=LGJbOhf-I5dL) zruQulv*mQ+``eGR#qA%mZ)TdN`!Rf? zqxg!`|J*lL>&zkhe~{VcxF#Z3{f!3X@_$SGz~cE{?!|-eSz_;6Vr-~q7kj#Ax3kv7 zLrs-X|7w{1z2|e&s!6mP7KcA`vX6Tw7KU`lmAEc&Ubk*Iac$ie$$IdO8=&BzyzJMQXA7F&N27S0eoSJ?mU9= z>+JVmIRiMm>lM7jg1esqJiU7fJP!<<2W%QV33zVs6!?|9&VrrZ^Gm?1d)@%g4~Lck zv%^b(p1rRDp56O8;Nv4_09*DgL*%FX_^hApKMNjp^f>;6T^k*AuG4wD!s)KDq>CRO km2*}T0IgX4gq;`*;D0W?FqXj%{vY_}VV!`3<$u8c13VWZ;Q#;t diff --git a/app/services/__pycache__/pls.cpython-314.pyc b/app/services/__pycache__/pls.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f14e2dca3da6914132a0d23eaa187ba8f6d213b0 GIT binary patch literal 5598 zcmcgwT})iZ6`r~G?%ieY{s7C52?o{}0~>7CUPx*a{{%0;F(vrokklc*+OTW0&U$w> z_mT!=RVZmBsy@VSQpHi#hDv#Whcg!1s3ciS`58+1DF_@C;N#(T8K&v zdWNWUSQruwv55dL3`;`}!!aZqGB1n6PQwZH(y(i&%qSaj8}1>+P=-8)XUJ=Kn}{mv zKGpG(UzJ zb%O;j21hgLgehxA=9NTxih{84he?7TXo%hm0y9r0$Qj|_&k2b`-A;0nPE(btf-0&~ zSb&{7z7n9DDNSeKv#4@YW(i}4rA-kkmI-W923cH)89GkTYzAW4Y*`} zV+G^iTw9tGL`%CI_8cHy!K7i6gk&be01z$6G?``i4JPp#eQG*!Az@0{m@yp^7>dnw zWMj~Y-Ph>6IO1@VEEx0{ zifK!lM5*y)kMsa~M&O*5)~`yXHLH%va(hHrRAqYwgRuVl=A&TTDB>;2o~4OEJL+Le zvfH!5gfIpNZ);MWwvp?bv@ip^S78NkAYjj!2BExpJxw58CmDGYe$T?MEJ!-6KA;Ut zMo!``9eI%JPLdRwNCW5_|4=(mi47!4gJg0Cp8|+X;&9Y=LVE-6ku3CiLQK+-YjjyK zoksFPA~R!{GC+6o?ChKpj?Ea;;Z){aG7Y#5fR>KIIdo-Xy8hD)i<{1uV^aXFaR5N+ zyq-y$p4b#1o^By-$ji_k8R7;5n}b6;Z5Y|tOr=+!O;1I+6Jg*u0a~*Nt$0`&;*nX- zVo5m4c#L&$9Wy#ZS7(an5{6mU|Jqa{i|!;OnzH4JOczV&S#U6b67iT3(^0g+U;!9p zjc+_6Z*t)}q0Q>v+4 zcaX-=J>qQiZ4l=U-@?#(Iq_DndiH$c*>lU&b|+B(?vcw!-afX_cc;4U-4`yua3#G` zy>DS)El{^nN2&%XOnECXa94oA8)BgX>Q({+|M>bb;;H_cP)C&s>lGpYr~W(G)a#9X z0s4Cd$R7kCSKIq2xweBw8(r7xMC5mNM_U}%%V{*|x?a%(d`|H~eGd9G%eguU`34F@ zbBz@FAjLk-oNtk%N6T`L)95~5uEUM|lLFF%eAHo_Fn1JZxj~h_a`A>ogqj;r-hyM(4XPK=8nKZQACEh%6M|h0I#1)>X@PPeqNlCqPvE<3rQ7AeIjC3`hmzS*|_*Ukl8Wecn~r_ zA5|UpSGMg}o{-0i$ZfBg6y*cxDvzNT+>&hheQ&({BVeTpzaIE0{}pDUGv?``m?@8% z-d@1R3`@@O=yuL|JP~6M3o~>EEC>XE(K(%n#S=^qxjFuDkTd0!738#I#}E>PLADo3 z2NHxkwhstcJvVqO=;4tpdjjfB5#k(%=F7rBbi~)zR)9Pk^dt?yk{yC-ZsUIj1|e^) za>v{4Z;j6Pf2NcJ)_v}8Tx?s~vs9Z89?1J5^Mh;NrbTJRyC?5yU5qVBA1cer)#yt5 z!F=1Hyt`*Ux-Q5M2glRzon8{JoL*@-vRZp&Vd##hYCGg@I16xg6Y$MXy!Cf|Sn+A? zk!?Wd+75Idz*b)OMW3XX=<#FHWd*q3wgb5)w?|vZwN4tPu4`ct`AALF>A3EqQPFk1 zY&Y_Iy-pc?kM+F$0qX2XyCsOE>b8y}%M|YLwyfj+w%lX{M*9l1Laa68} zkJ^Q^8Wf!JPz)=5QAT<{@w=@EeE3q0TJB+mwb**8)5|MvsKF zqtK~%tDtlkZWTzc1lD=J7_iEvC>HwjR9xZ!6z3sGm>_6A@y9T10O%<~o7EkuO3u*k z0wkWHd!eVDc9T;OhJgLkr-3;5Ef3<{r=Xq3xvf^rZtakV>c^moue}ABd2%PT_iEX# z(4mDxZyo!^u`6ARbVX@iJaAjtlb41*ngvqYfxR+Z2^aDYiNn8B3V!2zIgI1D0XrTS zX!l(LWU+&1+v9N*DfbZ5z-;vpr7IC&CfytXX2~WGuqg_+MQ!^J>%KYxf>z;j0l3%5 zqd8%&4uc>q6`M`4aN&j?wgRcqWs!q|=}zj&G}NW1FlGYJ3EJ6@n?qsRH!6S-tuWpW z#Lzgn5D<1aqEuApYYwbbL{^o^eBWB2;Y$32zSThJW+1fCotL_n3?O$!;_3QU_rRFf z{SkVZ?w5Ym6#37NNa0d$MFWV7r5J$qV)0RTszK4H*slu~EQJvzPqR+x3%Y|?o#mD4z1t53;R)X)+h}}u@lh9_l-I(sO{2)rIuS2;Z-I4P`&N{_8!+;MK^t$ zMx=L|y8q{Tn{#8(_9B|$OOI6F2<%{!`XZ>X2tJxXcD09>O3(m(dHTPquhY80S*KAn zUG`HD8-}%v!Xp6UD(eGcO1N?M46ygTWq7#$3NJ|PBsB9~Z&u#a}++zX`1QOw0LF;7`LXKoq^{qe5OL1 zGCdG#q>?Xp!jlW^1OV20m|+;bm~MFM>V&VJO`hX-H~xyluY1bM72QX_! z>%NDVGD{xl02smxH&(uBz$6taI3_)9c?{R#3z_&#Dshy(04;bzb==_J;TA;c8aen^ z()=aa_XY8MNn*E2?DOind3nLTLh3eTd}8~?2k$2v5eVEmmmhAhzQ~WeqI00dK}@E2x<5vJ5ab><%-t z*icnMb{?oAwRW4-NR1^P+!pso<*yUxC23ek2c+!>8U`y-vP-u8&;Xz%HX z^>tk|aWFyb$l8J!kcpSJzYEnYx{L;?6YnB44DYJY+jtC1cmPK1B2fcWP7YFF5&aqI zo16+ZO^%zKCL3uj8K6vBH#CzOS9BwQO>QKmO~ehI>{(-0#8ow@%i5HZko6#$X?vqd zoi;tOSUj0b8F537#WVr*o$%-tP|czp#NzrbQhi9jB+Y%s>TSa)vWy}EDLg#45Ms8+ zV9Od*4U_kbi0qnR=~~rcG{{Q$^{i9gz%e>arRf;el2r*~iMk3dxq)Zt9@d3ad>KdB zz4KP5w|lJEo^BNF;IJZGllAntA(HrLf&g(C z%7mOs8+|hc!MLgfr;Y~+9;Qv!YDfsFaV`rW8j0ut)(W7pNx}9y!Mg=|9~r2Gih#Zxd$&k+zvO=64=oGIudA)D=7sSD zJiCRgdW)dG10vN%eI**m5gpKej=;Z$LKH+Q6-8du^?QU=I!a4?z8xwPWg?Ek1QX{@RytnhK&c)ded~`u49mM4gAQBkT3DY2dOnrDDaZ)07|hP|0Vs z!kLqjb))57#m9!?iEHw(*w-c6j3Qo>r=dSIEe*+f6DSC-KTS!PZX>P%am9>jRW|9XsUee{ijSvd6CVfEz^njg(UR%x zypfPqLrEnyasz1xp)%R=)OA_Yh=T*X>9F}xH*wNrbXiVnhlpl?Xma*G+EMc6HK=rg zPcd68AeugMkmN{~GGh~QRci$$8+ZQ0j^8U-EM6;byu&Qj&aof6g*A8Oym4=4v2MM( zd1>VD)u%Tr#D$}aj-}Fvu18&eKKJF7n9L(+c z>6@En{`tv;y2X=AS0B}^9Owk4@D>Z>V1maxdw#0{6(9ZY3wOzX5C$Xv*>f=?ME@Fo zG2;2S;glvwr%e4dK zfL6(6D!{ooKlk{i>?x<4<2noP2dvzJnmL_CV%*9a#MFrRBZ93)3x&|+AoXJrE_+d) zg7iVO&<03_teLjVx5C)uEMVymkY$w}@&)@sUONScxBPJg1ncgHxGVckrmTL$SHIzH ze&%h?cv~{=))l7pnXmo}ojCVqaTERkH}bzMYr+AK>4?RKQwhjpfxwXIg0#0OkxGsz zqZ;{y8gY>YSPRCG(?H6ss2a)V?EHg3&N_&8l1Svfv(L((o=6R+$K|&)^8W;Jy50sA zL?Fh`QOBpq^AVCVNcxyRu)+IRdH;MO!`H05*0#>K&9a|!kUD?vg&TFJ4L-27?*9O8 CTDcVf literal 0 HcmV?d00001 diff --git a/app/services/ai.py b/app/services/ai.py index 1492893..ca1b9d5 100644 --- a/app/services/ai.py +++ b/app/services/ai.py @@ -30,6 +30,7 @@ async def answer(query: str, network_state: dict, alerts: list) -> str: def _rule_based(query: str, network_state: dict, alerts: list) -> str: q = query.lower() nfs = network_state.get("nfs", []) + cluster = network_state.get("cluster", {}) up = [n for n in nfs if n["state"] == "up"] down = [n for n in nfs if n["state"] == "down"] @@ -58,26 +59,40 @@ def _rule_based(query: str, network_state: dict, alerts: list) -> str: return _alerts_summary(alerts) if any(w in q for w in ["subscriber", "ue ", "device", "phone", "handset", "registration", "attach"]): - return _subscriber_analysis(nfs, alerts) + return _subscriber_analysis(nfs, alerts, cluster) if any(w in q for w in ["session", "pdu", "bearer", "user plane", "traffic", "throughput"]): - return _session_analysis(nfs, alerts) + return _session_analysis(nfs, alerts, cluster) # Default β†’ health summary - return _health_summary(up, down, alerts) + return _health_summary(up, down, alerts, cluster) -def _health_summary(up: list, down: list, alerts: list) -> str: +def _health_summary(up: list, down: list, alerts: list, cluster: dict) -> str: ts = datetime.now().strftime("%H:%M:%S") crit = [a for a in alerts if a.get("severity") == "critical"] warn = [a for a in alerts if a.get("severity") != "critical"] lines = [f"**P5G Network Health β€” {ts}**\n"] + nodes = cluster.get("nodes", []) if up: - lines.append(f"βœ… **{len(up)} UP**: {', '.join(n['name'] for n in up)}") + lines.append(f"βœ… **{len(up)} UP**: {', '.join(_nf_label(n) for n in up)}") if down: - lines.append(f"πŸ”΄ **{len(down)} DOWN**: {', '.join(n['name'] for n in down)}") - lines.append(f" ⚑ Action: check `{CONTAINER_RUNTIME} logs ` in the runtime host") + lines.append(f"πŸ”΄ **{len(down)} DOWN**: {', '.join(_nf_label(n) for n in down)}") + lines.append(" ⚑ Action: inspect the node shown for each affected NF before pulling logs.") + + if nodes: + lines.append(f"\n**Cluster nodes ({len(nodes)})**") + for node in nodes: + running = [nf["name"] for nf in node.get("nfs", []) if nf.get("state") == "up"] + down_nfs = [nf["name"] for nf in node.get("nfs", []) if nf.get("state") == "down"] + role = node.get("role", "AP") + lines.append( + f"β€’ **{node['hostname']}** ({role}{', local' if node.get('current') else ''})" + f" β€” running: {', '.join(running) or 'none'}" + ) + if down_nfs: + lines.append(f" down here: {', '.join(down_nfs)}") if alerts: lines.append(f"\n⚠️ **{len(alerts)} alert(s)** β€” {len(crit)} critical, {len(warn)} warning") @@ -102,8 +117,15 @@ def _nf_detail(nf_name: str, nfs: list, alerts: list) -> str: f"Check: `{CONTAINER_RUNTIME} ps | grep {nf_name.lower()}`") icon = "βœ…" if nf["state"] == "up" else "πŸ”΄" - lines = [f"{icon} **{nf_name}** is **{nf['state'].upper()}**", - f"Instance: `{nf.get('instance', 'n/a')}`"] + placements = nf.get("nodes", []) + lines = [f"{icon} **{nf_name}** is **{nf['state'].upper()}**"] + if placements: + node_text = ", ".join( + f"{node['hostname']} ({'/'.join(node.get('roles', []))})" + for node in placements + ) + lines.append(f"Nodes: {node_text}") + lines.append(f"Instance: `{nf.get('instance', 'n/a')}`") if nf_alerts: lines.append(f"\n⚠️ {len(nf_alerts)} alert(s) for {nf_name}:") for a in nf_alerts: @@ -129,43 +151,72 @@ def _alerts_summary(alerts: list) -> str: return "\n".join(lines) -def _subscriber_analysis(nfs: list, alerts: list) -> str: +def _subscriber_analysis(nfs: list, alerts: list, cluster: dict) -> str: amf = next((n for n in nfs if n["name"] == "AMF"), None) smf = next((n for n in nfs if n["name"] == "SMF"), None) lines = ["**Subscriber & Registration Analysis**\n"] - lines.append(f"AMF (registration/mobility): {'βœ… UP' if amf and amf['state'] == 'up' else 'πŸ”΄ DOWN β€” subscribers cannot register'}") - lines.append(f"SMF (session management): {'βœ… UP' if smf and smf['state'] == 'up' else 'πŸ”΄ DOWN β€” no new data sessions'}") + lines.append(f"AMF (registration/mobility): {_nf_sentence(amf, 'subscribers cannot register')}") + lines.append(f"SMF (session management): {_nf_sentence(smf, 'no new data sessions')}") sub_alerts = [a for a in alerts if any(k in a.get("name", "").lower() for k in ["ue", "subscriber", "session", "attach", "registration"])] if sub_alerts: lines.append(f"\n⚠️ {len(sub_alerts)} subscriber-related alert(s) active.") else: lines.append("\nNo subscriber-related alerts detected.") + lines.append(_cluster_scope(cluster)) return "\n".join(lines) -def _session_analysis(nfs: list, alerts: list) -> str: +def _session_analysis(nfs: list, alerts: list, cluster: dict) -> str: smf = next((n for n in nfs if n["name"] == "SMF"), None) upf = next((n for n in nfs if n["name"] == "UPF"), None) lines = ["**PDU Session & Data Plane Analysis**\n"] - lines.append(f"SMF: {'βœ… UP' if smf and smf['state'] == 'up' else 'πŸ”΄ DOWN'}") - lines.append(f"UPF: {'βœ… UP' if upf and upf['state'] == 'up' else 'πŸ”΄ DOWN'}") + lines.append(f"SMF: {_nf_sentence(smf, 'session setup is blocked')}") + lines.append(f"UPF: {_nf_sentence(upf, 'user-plane forwarding is blocked')}") if (not smf or smf["state"] != "up") or (not upf or upf["state"] != "up"): lines.append("\n⚑ **Impact**: PDU sessions will fail until both SMF and UPF are operational.") else: lines.append("\nBoth SMF and UPF operational β€” sessions should be establishing normally.") + lines.append(_cluster_scope(cluster)) return "\n".join(lines) +def _nf_label(nf: dict) -> str: + placements = nf.get("nodes", []) + if not placements: + return nf["name"] + return f"{nf['name']} on {', '.join(node['hostname'] for node in placements)}" + + +def _nf_sentence(nf: dict | None, impact: str) -> str: + if not nf: + return "β—‹ N/A" + if nf.get("state") == "up": + nodes = ", ".join(node["hostname"] for node in nf.get("nodes", [])) or nf.get("instance", "unknown host") + return f"βœ… UP on {nodes}" + return f"πŸ”΄ DOWN β€” {impact}" + + +def _cluster_scope(cluster: dict) -> str: + nodes = cluster.get("nodes", []) + if not nodes: + return "\nCluster discovery is not available." + details = ", ".join(f"{node['hostname']} ({node.get('role', 'AP')})" for node in nodes) + return f"\nCluster scope checked: {details}" + + # ── LLM backends ────────────────────────────────────────────────────────── def _build_context(network_state: dict, alerts: list) -> str: nfs = network_state.get("nfs", []) up = [n["name"] for n in nfs if n["state"] == "up"] down = [n["name"] for n in nfs if n["state"] == "down"] + nodes = network_state.get("cluster", {}).get("nodes", []) + node_summary = ", ".join(f"{node['hostname']} ({node.get('role', 'AP')})" for node in nodes) or "none" return ( f"NFs UP: {', '.join(up) or 'none'}\n" f"NFs DOWN: {', '.join(down) or 'none'}\n" + f"Cluster nodes: {node_summary}\n" f"Active alerts: {', '.join(a.get('name','') for a in alerts[:5]) or 'none'}" ) diff --git a/app/services/alertmanager.py b/app/services/alertmanager.py index 72ba5d2..c3c02ed 100644 --- a/app/services/alertmanager.py +++ b/app/services/alertmanager.py @@ -2,6 +2,7 @@ import httpx from app.config import ALERTMANAGER_URL +from app.services import cluster_inventory _BASE = ALERTMANAGER_URL.rstrip("/") @@ -16,14 +17,29 @@ async def get_alerts() -> list: except Exception: return [] + cluster = await cluster_inventory.get_cluster_inventory() alerts = [] for a in raw: labels = a.get("labels", {}) annotations = a.get("annotations", {}) + name = labels.get("alertname", "Unknown") + summary = annotations.get("summary", annotations.get("description", "")) + nf_name = _infer_nf(name, summary, labels.get("instance", "")) + nodes = cluster_inventory.find_nf_nodes(cluster, nf_name) if nf_name else [] alerts.append({ - "name": labels.get("alertname", "Unknown"), + "name": name, "severity": labels.get("severity", "warning"), "instance": labels.get("instance", ""), - "summary": annotations.get("summary", annotations.get("description", "")), + "summary": summary, + "nf": nf_name, + "nodes": nodes, }) return alerts + + +def _infer_nf(name: str, summary: str, instance: str) -> str: + text = f"{name} {summary} {instance}".upper() + for nf_name in ["AMF", "SMF", "UPF", "UDM", "UDR", "NRF", "AUSF", "PCF", "MME", "SGWC", "DRA", "DSM"]: + if nf_name in text: + return nf_name + return "" diff --git a/app/services/cluster_inventory.py b/app/services/cluster_inventory.py new file mode 100644 index 0000000..7451cca --- /dev/null +++ b/app/services/cluster_inventory.py @@ -0,0 +1,180 @@ +"""Cluster discovery built on top of the PLS API.""" + +from __future__ import annotations + +import asyncio +import re + +from app.config import ALL_NFS +from app.services import pls, prometheus + +ROLE_NF_MAP = { + "5GALL": {"amf", "smf", "pcf", "udr", "udm", "nrf", "eir", "ausf", "dra", "upf", "chf", "smsf", "aaa", "bmsc"}, + "CP": {"amf", "smf", "pcf", "udr", "udm", "nrf", "eir", "ausf", "dra", "chf", "smsf", "aaa", "bmsc"}, + "UP": {"upf"}, + "DCP": {"amf", "smf", "pcf", "chf", "smsf", "bmsc"}, + "DLF": {"udr", "udm", "nrf", "eir", "ausf", "aaa"}, + "SIG": {"dra"}, + "4GALL": {"mme", "sgwc", "smf", "pcf", "chf", "udr", "udm", "nrf", "eir", "ausf", "dra", "upf", "smsf", "aaa", "bmsc"}, + "4GCP": {"mme", "sgwc", "smf", "pcf", "chf", "udr", "udm", "nrf", "eir", "ausf", "dra", "smsf", "aaa", "bmsc"}, + "4GDCP": {"mme", "sgwc", "smf", "pcf", "chf", "smsf", "bmsc"}, + "COMBOALL": {"amf", "mme", "sgwc", "smf", "pcf", "chf", "udr", "udm", "nrf", "eir", "ausf", "dra", "upf", "smsf", "aaa", "bmsc"}, + "COMBOCP": {"amf", "mme", "sgwc", "smf", "pcf", "chf", "udr", "udm", "nrf", "eir", "ausf", "dra", "smsf", "aaa", "bmsc"}, + "COMBODCP": {"amf", "mme", "sgwc", "smf", "pcf", "chf", "aaa"}, +} +ROLE_ALIASES = { + "UPF": "UP", +} +ROLE_PRIORITY = ["COMBOALL", "COMBOCP", "COMBODCP", "5GALL", "4GALL", "4GCP", "4GDCP", "DCP", "DLF", "SIG", "CP", "UP"] + + +def _infer_role(hostname: str) -> str: + tokens = [t for t in re.split(r"[^A-Za-z0-9]+", hostname.upper()) if t] + normalized = [ROLE_ALIASES.get(token, token) for token in tokens] + for role in ROLE_PRIORITY: + if role in normalized: + return role + for token in normalized: + if token.endswith("UPF"): + return "UP" + return "AP" + + +async def get_cluster_inventory() -> dict: + cluster = await pls.get_cluster_status() + if not cluster: + return { + "enabled": False, + "current_node": None, + "fully_established": False, + "nodes": [], + } + + node_names = [node.get("name", "") for node in cluster.get("nodes", [])] + info_tasks = [asyncio.create_task(pls.get_system_info(pls.node_host(name))) for name in node_names] + service_tasks = [asyncio.create_task(pls.get_services(pls.node_host(name))) for name in node_names] + infos = await asyncio.gather(*info_tasks, return_exceptions=True) + services = await asyncio.gather(*service_tasks, return_exceptions=True) + + nodes: list[dict] = [] + for idx, node in enumerate(cluster.get("nodes", [])): + info = infos[idx] if isinstance(infos[idx], dict) else {} + node_services = services[idx] if isinstance(services[idx], list) else [] + started = {svc["name"] for svc in node_services if svc.get("state") == "started"} + hostname = info.get("hostname") or pls.node_host(node.get("name", "")) + role = _infer_role(hostname) + nodes.append( + { + "name": node.get("name", ""), + "address": pls.node_host(node.get("name", "")), + "hostname": hostname, + "current": node.get("name") == cluster.get("current_node"), + "repositories": node.get("repositories", []), + "role": role, + "roles": [role], + "expected_nfs": sorted(ROLE_NF_MAP.get(role, set())), + "services": node_services, + "started_services": sorted(started), + } + ) + + return { + "enabled": True, + "current_node": cluster.get("current_node"), + "fully_established": bool(cluster.get("fully_established")), + "nodes": nodes, + } + + +def _aggregate_nf_state(nf_name: str, nodes: list[dict], prom_states: dict[str, dict]) -> dict: + service_name = nf_name.lower() + placements = [] + seen_service = False + for node in nodes: + for service in node.get("services", []): + if service.get("name") != service_name: + continue + seen_service = True + if service.get("state") == "started": + placements.append( + { + "hostname": node["hostname"], + "address": node["address"], + "roles": node["roles"], + } + ) + + prom_state = prom_states.get(nf_name, {"state": "unknown", "instance": ""}) + if placements: + state = prom_state["state"] if prom_state["state"] in {"up", "down"} else "up" + instance = ", ".join(p["hostname"] for p in placements) + elif seen_service: + state = "down" + instance = "" + else: + state = prom_state["state"] + instance = prom_state["instance"] + + return { + "name": nf_name, + "state": state, + "instance": instance, + "nodes": placements, + } + + +def _node_nf_state(node: dict, nf_name: str) -> dict: + service_name = nf_name.lower() + service = next((svc for svc in node.get("services", []) if svc.get("name") == service_name), None) + if not service: + return {"name": nf_name, "state": "unknown"} + if service.get("state") == "started": + return {"name": nf_name, "state": "up"} + return {"name": nf_name, "state": "down"} + + +def _attach_node_nf_status(nodes: list[dict]) -> list[dict]: + enriched = [] + for node in nodes: + node_copy = dict(node) + expected_nfs = node_copy.get("expected_nfs", []) + node_copy["nfs"] = [_node_nf_state(node_copy, nf_name.upper()) for nf_name in expected_nfs] + enriched.append(node_copy) + return enriched + + +async def get_network_status() -> dict: + inventory_task = asyncio.create_task(get_cluster_inventory()) + prom_task = asyncio.create_task(prometheus.get_nf_status_map()) + inventory, prom_states = await asyncio.gather(inventory_task, prom_task) + + nodes = _attach_node_nf_status(inventory.get("nodes", [])) + inventory["nodes"] = nodes + nfs = [_aggregate_nf_state(nf_name, nodes, prom_states) for nf_name in ALL_NFS] + up = sum(1 for nf in nfs if nf["state"] == "up") + down = sum(1 for nf in nfs if nf["state"] == "down") + + return { + "nfs": nfs, + "summary": {"up": up, "down": down, "total": len(nfs)}, + "cluster": inventory, + } + + +def find_nf_nodes(cluster: dict, nf_name: str) -> list[dict]: + nodes = cluster.get("nodes", []) + matches = [] + for node in nodes: + for nf in node.get("nfs", []): + if nf.get("name") == nf_name: + matches.append( + { + "hostname": node["hostname"], + "address": node["address"], + "role": node.get("role", "AP"), + "current": node.get("current", False), + "state": nf.get("state", "unknown"), + } + ) + break + return matches diff --git a/app/services/log_analyzer.py b/app/services/log_analyzer.py index 63915b6..6b4a95c 100644 --- a/app/services/log_analyzer.py +++ b/app/services/log_analyzer.py @@ -235,20 +235,23 @@ async def analyze_logs() -> dict: Gather log-pattern issues + Prometheus NF status + Alertmanager alerts. Returns a fully structured dict ready for JSON serialisation. """ - from app.services import alertmanager, prometheus + from app.services import alertmanager, prometheus, cluster_inventory # Kick off all I/O in parallel containers_f = asyncio.create_task(_discover_containers()) alerts_f = asyncio.create_task(alertmanager.get_alerts()) nf_status_f = asyncio.create_task(prometheus.get_nf_status()) + cluster_f = asyncio.create_task(cluster_inventory.get_cluster_inventory()) containers = await containers_f - alerts, nf_statuses = await asyncio.gather(alerts_f, nf_status_f, + alerts, nf_statuses, cluster = await asyncio.gather(alerts_f, nf_status_f, cluster_f, return_exceptions=True) if isinstance(alerts, Exception): alerts = [] if isinstance(nf_statuses, Exception): nf_statuses = [] + if isinstance(cluster, Exception): + cluster = {"enabled": False, "nodes": []} # Read all container logs concurrently log_tasks = {nf: asyncio.create_task(_read_logs(cname)) @@ -280,25 +283,29 @@ async def analyze_logs() -> dict: # 2. NF-down events from Prometheus for nf_st in nf_statuses: if isinstance(nf_st, dict) and nf_st.get("state") == "down": + node_text = ", ".join(node["hostname"] for node in nf_st.get("nodes", [])) issues.append({ "id": f"nf-down-{nf_st['name']}", "category": "Connectivity", "nf": nf_st["name"], + "node": node_text, "severity": "critical", "count": 1, "description": f"{nf_st['name']} is unreachable", - "remediation": (f"Run `{CONTAINER_RUNTIME} ps` and check if {nf_st['name']} " - f"container is running; inspect its logs."), + "remediation": (f"Check {node_text or 'the hosting node'} first, then run " + f"`{CONTAINER_RUNTIME} ps` and inspect `{nf_st['name'].lower()}` logs."), "source": "prometheus", }) # 3. Active Alertmanager alerts for alert in alerts: if isinstance(alert, dict): + node_text = ", ".join(node["hostname"] for node in alert.get("nodes", [])) issues.append({ "id": f"alert-{alert.get('name', '')}-{len(issues)}", "category": _alert_category(alert), "nf": _alert_nf(alert), + "node": node_text, "severity": alert.get("severity", "warning"), "count": 1, "description": alert.get("summary") or alert.get("name", "Unknown alert"), @@ -331,6 +338,7 @@ async def analyze_logs() -> dict: "categories": categories, "timestamp": datetime.now().isoformat(), "log_sources": list(containers.keys()), + "cluster": cluster, } # Persist to history ring-buffer diff --git a/app/services/pls.py b/app/services/pls.py new file mode 100644 index 0000000..8e62242 --- /dev/null +++ b/app/services/pls.py @@ -0,0 +1,78 @@ +"""PLS API client for cluster and per-node discovery.""" + +from __future__ import annotations + +from urllib.parse import urlsplit, urlunsplit + +import httpx + +from app.config import PLS_AUTH_BACKEND, PLS_BASE_URL, PLS_PASSWORD, PLS_USERNAME, PLS_VERIFY_TLS + +_token: str | None = None + + +def _base_url_for_host(host: str | None = None) -> str: + if not host: + return PLS_BASE_URL.rstrip("/") + parts = urlsplit(PLS_BASE_URL) + return urlunsplit((parts.scheme, host, parts.path.rstrip("/"), "", "")) + + +async def _login() -> str | None: + global _token + if _token: + return _token + if not PLS_USERNAME or not PLS_PASSWORD: + return None + + try: + async with httpx.AsyncClient(timeout=5, verify=PLS_VERIFY_TLS) as client: + response = await client.post( + f"{_base_url_for_host()}/auth/login", + json={ + "username": PLS_USERNAME, + "password": PLS_PASSWORD, + "backend": PLS_AUTH_BACKEND, + }, + ) + response.raise_for_status() + data = response.json() + _token = data.get("access_token") + return _token + except Exception: + return None + + +async def _get(path: str, host: str | None = None) -> dict | list | None: + token = await _login() + if not token: + return None + + headers = {"Authorization": f"Bearer {token}"} + url = f"{_base_url_for_host(host)}/{path.lstrip('/')}" + try: + async with httpx.AsyncClient(timeout=5, verify=PLS_VERIFY_TLS) as client: + response = await client.get(url, headers=headers) + response.raise_for_status() + return response.json() + except Exception: + return None + + +def node_host(node_name: str) -> str: + return node_name.split("@", 1)[1] if "@" in node_name else node_name + + +async def get_cluster_status() -> dict | None: + data = await _get("data_layer/cluster/status") + return data if isinstance(data, dict) else None + + +async def get_system_info(host: str | None = None) -> dict | None: + data = await _get("system/info", host=host) + return data if isinstance(data, dict) else None + + +async def get_services(host: str | None = None) -> list[dict]: + data = await _get("services", host=host) + return data if isinstance(data, list) else [] diff --git a/app/services/prometheus.py b/app/services/prometheus.py index 96d863b..3b89324 100644 --- a/app/services/prometheus.py +++ b/app/services/prometheus.py @@ -14,12 +14,12 @@ async def query(promql: str) -> list: return r.json()["data"]["result"] -async def get_nf_status() -> list: - """Return a list of {name, state, instance} for every known NF.""" +async def get_nf_status_map() -> dict[str, dict]: + """Return Prometheus-backed NF status keyed by display name.""" try: results = await query("up") except Exception: - return [{"name": n, "state": "unknown", "instance": ""} for n in ALL_NFS] + return {n: {"name": n, "state": "unknown", "instance": ""} for n in ALL_NFS} seen: dict[str, dict] = {} for r in results: @@ -38,4 +38,9 @@ async def get_nf_status() -> list: if n not in seen: seen[n] = {"name": n, "state": "unknown", "instance": ""} - return list(seen.values()) + return seen + + +async def get_nf_status() -> list: + """Return a list of {name, state, instance} for every known NF.""" + return list((await get_nf_status_map()).values()) diff --git a/app/ui/actions.html b/app/ui/actions.html index 451d783..8dc36d3 100644 --- a/app/ui/actions.html +++ b/app/ui/actions.html @@ -155,6 +155,12 @@ body { background: rgba(255,255,255,0.07); color: var(--text); width: fit-content; white-space: nowrap; } +.issue-node { + font-size: 10px; font-weight: 600; letter-spacing: 0.04em; + padding: 2px 7px; border-radius: 5px; margin-top: 5px; + background: rgba(59,130,246,0.12); color: var(--blue); + width: fit-content; white-space: nowrap; +} .issue-body {} .issue-desc { font-size: 13px; font-weight: 500; line-height: 1.4; } .issue-rem { font-size: 11px; color: var(--muted); margin-top: 3px; line-height: 1.4; } @@ -469,6 +475,7 @@ function renderDetail(cat) {
${esc(iss.nf)}
${esc(iss.description)}
+ ${iss.node ? `
${esc(iss.node)}
` : ''}
β€· ${esc(iss.remediation||'')}
${esc(iss.source||'log')}
diff --git a/app/ui/index.html b/app/ui/index.html index 4c20bff..e7d149e 100644 --- a/app/ui/index.html +++ b/app/ui/index.html @@ -60,8 +60,10 @@ header h1 span { color: var(--muted); font-weight: 400; } /* ── Left panel ─────────────────────────────────────────────────── */ .left { background: var(--surface); border-right: 1px solid var(--border); - display: flex; flex-direction: column; overflow: hidden; + display: flex; flex-direction: column; overflow-y: auto; overflow-x: hidden; } +.left::-webkit-scrollbar { width: 5px; } +.left::-webkit-scrollbar-thumb { background: var(--border); border-radius: 4px; } .section { padding: 14px 16px; border-bottom: 1px solid var(--border); } .section-title { font-size: 10px; font-weight: 700; text-transform: uppercase; @@ -88,6 +90,79 @@ header h1 span { color: var(--muted); font-weight: 400; } .nf-card.up .nf-state { color: var(--green); } .nf-card.down .nf-state { color: var(--red); } +/* Cluster nodes */ +.node-list { display: flex; flex-direction: column; gap: 8px; } +.node-card { + background: var(--card); border: 1px solid var(--border); border-radius: 10px; + overflow: hidden; +} +.node-summary { + display: flex; align-items: center; gap: 8px; padding: 10px 12px; cursor: pointer; +} +.node-summary:hover { background: rgba(255,255,255,.02); } +.node-top { display: flex; align-items: center; gap: 8px; width: 100%; } +.node-name { font-size: 13px; font-weight: 700; } +.node-addr { font-size: 11px; color: var(--muted); margin-top: 3px; } +.node-caret { + margin-left: 8px; font-size: 11px; color: var(--muted); transition: transform .15s; +} +.node-card.open .node-caret { transform: rotate(180deg); } +.node-role { + margin-left: auto; font-size: 10px; font-weight: 700; letter-spacing: .08em; + border-radius: 999px; padding: 3px 8px; border: 1px solid var(--border); + color: var(--blue); background: rgba(59,130,246,.12); +} +.node-role.current { + color: var(--green); border-color: rgba(16,185,129,.5); background: rgba(16,185,129,.12); +} +.node-meta { + display: flex; flex-wrap: wrap; gap: 6px; margin-top: 8px; +} +.node-chip { + font-size: 10px; color: var(--muted); padding: 2px 7px; + border-radius: 999px; background: rgba(255,255,255,.04); border: 1px solid var(--border); +} +.node-services { + margin-top: 8px; font-size: 11px; color: var(--text); line-height: 1.4; +} +.node-services b, +.node-profile b { + color: var(--muted); font-weight: 600; +} +.node-profile { + margin-top: 6px; font-size: 11px; color: var(--text); line-height: 1.4; +} +.node-empty { + color: var(--muted); font-size: 12px; +} +.node-details { + display: none; padding: 0 12px 12px; border-top: 1px solid rgba(255,255,255,.04); +} +.node-card.open .node-details { display: block; } +.node-nf-grid { + display: grid; grid-template-columns: repeat(3, 1fr); gap: 6px; margin-top: 10px; +} +.node-nf { + background: rgba(255,255,255,.03); + border: 1px solid var(--border); + border-radius: 8px; + padding: 7px 5px; + border-left: 3px solid var(--border); + text-align: center; +} +.node-nf.up { border-left-color: var(--green); } +.node-nf.down { border-left-color: var(--red); } +.node-nf.unknown { border-left-color: var(--yellow); } +.node-nf-name { + font-size: 10px; font-weight: 700; color: var(--text); letter-spacing: .04em; +} +.node-nf-state { + margin-top: 3px; font-size: 9px; text-transform: uppercase; letter-spacing: .06em; color: var(--muted); +} +.node-nf.up .node-nf-state { color: var(--green); } +.node-nf.down .node-nf-state { color: var(--red); } +.node-nf.unknown .node-nf-state { color: var(--yellow); } + /* Alerts panel */ .alerts-scroll { flex: 1; overflow-y: auto; padding: 14px 16px; } .alerts-scroll::-webkit-scrollbar { width: 4px; } @@ -101,6 +176,7 @@ header h1 span { color: var(--muted); font-weight: 400; } .alert-row.critical { border-left-color: var(--red); } .alert-row-name { font-size: 12px; font-weight: 600; } .alert-row-desc { font-size: 11px; color: var(--muted); margin-top: 2px; } +.alert-row-node { font-size: 10px; color: var(--blue); margin-top: 5px; } /* ── Chat panel ─────────────────────────────────────────────────── */ .chat { display: flex; flex-direction: column; overflow: hidden; } @@ -185,13 +261,19 @@ header h1 span { color: var(--muted); font-weight: 400; }
- Network Functions + Cluster Overview
Β·Β·Β·
+
+
Discovered Nodes
+
+
Loading cluster inventory…
+
+
Active Alerts
Loading…
@@ -221,6 +303,15 @@ header h1 span { color: var(--muted); font-weight: 400; } // ── Utilities ────────────────────────────────────────────────────────────── const $ = id => document.getElementById(id); const ts = () => new Date().toLocaleTimeString([],{hour:'2-digit',minute:'2-digit'}); +const ROLE_LABELS = { + '5GALL': '5G All', + '4GALL': '4G All', + '4GCP': '4G CP', + '4GDCP': '4G DCP', + 'COMBOALL': 'Combo All', + 'COMBOCP': 'Combo CP', + 'COMBODCP': 'Combo DCP', +}; function md(text) { // minimal markdown: **bold**, `code`, newlines @@ -261,15 +352,70 @@ async function loadNFs() {
${nf.state==='up'?'● up':nf.state==='down'?'● dn':'β—‹ n/a'}
`; grid.appendChild(c); }); + renderNodes(d.cluster); $('dot').className = 'dot'; $('connLabel').textContent = 'Live'; } catch { $('dot').className = 'dot err'; $('connLabel').textContent = 'Unreachable'; $('nfGrid').innerHTML = '
Cannot reach API
'; + $('nodeList').innerHTML = '
Cannot reach cluster discovery API
'; } } +function toggleNodeCard(button) { + button.closest('.node-card')?.classList.toggle('open'); +} + +function renderNodes(cluster) { + const list = $('nodeList'); + const nodes = cluster?.nodes || []; + if (!nodes.length) { + list.innerHTML = '
No cluster nodes discovered
'; + return; + } + + list.innerHTML = nodes.map(node => { + const role = ROLE_LABELS[node.role] || node.role || 'AP'; + const repoChips = (node.repositories || []).slice(0, 3).map(repo => + `${repo.type}:${repo.role}` + ).join(''); + const running = (node.started_services || []).filter(name => !['alertmanager','prometheus','ncm','pls','fluent-bit','grafana','openvpn','ssh','node-exporter','podman-exporter','licensed','webconsole'].includes(name)); + const serviceText = running.length ? running.join(', ') : 'No managed NFs started'; + const expected = (node.expected_nfs || []).join(', ') || 'No NF profile mapped'; + const nfTiles = (node.nfs || []).map(nf => ` +
+
${nf.name}
+
${nf.state === 'up' ? '● up' : nf.state === 'down' ? '● dn' : 'β—‹ n/a'}
+
+ `).join(''); + const downCount = (node.nfs || []).filter(nf => nf.state === 'down').length; + const openClass = node.current ? 'open' : ''; + return ` +
+
+
+
+
${node.hostname}
+
${node.address} Β· ${node.nfs.filter(nf => nf.state === 'up').length} up${downCount ? `, ${downCount} down` : ''}
+
+
${role}${node.current ? ' Β· local' : ''}
+
β–Ύ
+
+
+
+
+ ${repoChips || 'No repo data'} +
+
Running: ${serviceText}
+
Profile: ${expected}
+
${nfTiles || '
No node-scoped NF data
'}
+
+
+ `; + }).join(''); +} + async function loadAlerts() { try { const d = await (await fetch('./api/alerts')).json(); @@ -281,6 +427,7 @@ async function loadAlerts() { `
${a.name}
${a.summary||a.instance||''}
+
${(a.nodes||[]).length ? 'Node: ' + a.nodes.map(n => n.hostname).join(', ') : 'Node: unresolved'}
` ).join(''); } diff --git a/app/ui/tasks.html b/app/ui/tasks.html index f87dd7c..9ed6f14 100644 --- a/app/ui/tasks.html +++ b/app/ui/tasks.html @@ -59,6 +59,22 @@ header h1 span { color: var(--muted); font-weight: 400; } } .main::-webkit-scrollbar { width: 5px; } .main::-webkit-scrollbar-thumb { background: var(--border); border-radius: 4px; } +.content-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) 420px; + gap: 24px; + align-items: start; +} +.tasks-col { + display: flex; + flex-direction: column; + gap: 24px; + min-width: 0; +} +.log-col { + position: sticky; + top: 0; +} /* ── Section headers ─────────────────────────────────────────────── */ .section-title { @@ -171,6 +187,15 @@ header h1 span { color: var(--muted); font-weight: 400; } } .modal-confirm.danger { background: var(--red); } .modal-confirm.warning { background: var(--yellow); color: #000; } + +@media (max-width: 1100px) { + .content-grid { + grid-template-columns: 1fr; + } + .log-col { + position: static; + } +} @@ -183,42 +208,46 @@ header h1 span { color: var(--muted); font-weight: 400; }
- - -
-
Diagnostics & Health
-
-
- - -
-
Network Operations
-
-
- - -
-
Maintenance
-
-
- - -
-
-
- β–Έ Run Log - 0 entries +
+
+ +
+
Diagnostics & Health
+
-
- - + + +
+
Network Operations
+
+
+ + +
+
Maintenance
+
-
-
No actions run yet.
+ +
+ +
+
+
+ β–Έ Run Log + 0 entries +
+
+ + +
+
+
+
No actions run yet.
+
+
-
@@ -255,6 +284,17 @@ const ACTIONS = { ], }; +function nfNodeLabel(nf) { + const nodes = nf?.nodes || []; + return nodes.length ? nodes.map(n => n.hostname).join(', ') : 'unresolved node'; +} + +async function fetchNetworkStatus() { + const r = await fetch('/api/network/status'); + if (!r.ok) throw new Error('HTTP ' + r.status); + return await r.json(); +} + // ── Render cards ────────────────────────────────────────────────────────── function renderGrid(gridId, items) { const g = document.getElementById(gridId); @@ -289,6 +329,7 @@ function handleAction(id) { const all = [...ACTIONS.diag, ...ACTIONS.ops, ...ACTIONS.maint]; const a = all.find(x => x.id === id); if (!a) return; + revealLogPanel(true); if (a.safe) { a.run(); return; } pendingAction = a; document.getElementById('modalTitle').textContent = a.name; @@ -306,6 +347,7 @@ function closeModal() { function runConfirmed() { closeModal(); + revealLogPanel(true); if (pendingAction) { pendingAction.run(); pendingAction = null; } } @@ -326,6 +368,17 @@ function addLog(msg, type='info') { renderLog(); } +function revealLogPanel(forceExpand=false) { + const panel = document.getElementById('logPanel'); + const el = document.getElementById('logBody'); + const btn = document.getElementById('expandBtn'); + if (forceExpand && !el.classList.contains('expanded')) { + el.classList.add('expanded'); + btn.textContent = '‑ Collapse'; + } + panel.scrollIntoView({ behavior: 'smooth', block: 'start' }); +} + function renderLog() { const el = document.getElementById('logBody'); document.getElementById('logEmpty').style.display = logLines.length ? 'none' : ''; @@ -368,17 +421,16 @@ document.addEventListener('DOMContentLoaded', () => { // ── Action implementations ───────────────────────────────────────────────── async function pingNFs() { - addLog('β–Έ Pinging all NFs via Prometheus endpoint…', 'run'); + addLog('β–Έ Checking all discovered NFs across cluster nodes…', 'run'); try { - const r = await fetch('/api/network/nf-status'); - const d = await r.json(); - const nfs = d.nf_status || []; + const d = await fetchNetworkStatus(); + const nfs = d.nfs || []; const up = nfs.filter(n => n.state === 'up').length; const down = nfs.filter(n => n.state === 'down').length; - nfs.forEach(n => addLog(` ${n.name}: ${n.state.toUpperCase()}`, n.state === 'up' ? 'ok' : 'err')); - addLog(`βœ“ Ping complete β€” ${up} up, ${down} down`, down > 0 ? 'warn' : 'ok'); + nfs.forEach(n => addLog(` ${n.name}: ${n.state.toUpperCase()} on ${nfNodeLabel(n)}`, n.state === 'up' ? 'ok' : n.state === 'down' ? 'err' : 'warn')); + addLog(`βœ“ Cluster check complete β€” ${up} up, ${down} down`, down > 0 ? 'warn' : 'ok'); } catch(e) { - addLog('βœ— Failed to reach Prometheus: ' + e.message, 'err'); + addLog('βœ— Failed to reach network status API: ' + e.message, 'err'); } } @@ -392,7 +444,7 @@ async function refreshAlerts() { addLog('βœ“ No active alerts β€” network is healthy', 'ok'); } else { addLog(`⚠ ${alerts.length} active alert(s):`, 'warn'); - alerts.forEach(a => addLog(` [${(a.labels?.severity||'info').toUpperCase()}] ${a.labels?.alertname||'Unknown'}`, 'warn')); + alerts.forEach(a => addLog(` [${(a.severity||'info').toUpperCase()}] ${a.name} on ${(a.nodes||[]).map(n => n.hostname).join(', ') || 'unresolved node'}`, 'warn')); } } catch(e) { addLog('βœ— Failed to reach Alertmanager: ' + e.message, 'err'); @@ -400,15 +452,18 @@ async function refreshAlerts() { } async function nfReport() { - addLog('β–Έ Generating full NF status report…', 'run'); + addLog('β–Έ Generating cluster-wide NF status report…', 'run'); try { - const r = await fetch('/api/network/nf-status'); - const d = await r.json(); - const nfs = d.nf_status || []; + const d = await fetchNetworkStatus(); + const nfs = d.nfs || []; const up = nfs.filter(n => n.state === 'up').length; addLog(`βœ“ Report: ${up}/${nfs.length} NFs operational`, up === nfs.length ? 'ok' : 'warn'); + (d.cluster?.nodes || []).forEach(node => { + const running = (node.nfs || []).filter(nf => nf.state === 'up').map(nf => nf.name); + addLog(` ${node.hostname} (${node.role}): ${running.join(', ') || 'no active NFs'}`, 'info'); + }); addLog(` Timestamp: ${new Date().toISOString()}`, 'info'); - addLog(` Source: Prometheus metrics`, 'info'); + addLog(` Source: PLS cluster discovery + Prometheus`, 'info'); } catch(e) { addLog('βœ— Report generation failed: ' + e.message, 'err'); } @@ -453,16 +508,15 @@ async function emulatedSession() { } async function checkDevices() { - addLog('β–Έ Fetching connected device list…', 'run'); + addLog('β–Έ Checking cluster nodes for subscriber-serving NFs…', 'run'); try { - const r = await fetch('/api/network/nf-status'); - const d = await r.json(); - const nfs = d.nf_status || []; + const d = await fetchNetworkStatus(); + const nfs = d.nfs || []; const amf = nfs.find(n => n.name === 'AMF'); - addLog(` AMF state: ${amf ? amf.state.toUpperCase() : 'UNKNOWN'}`, amf?.state === 'up' ? 'ok' : 'warn'); + addLog(` AMF state: ${amf ? amf.state.toUpperCase() : 'UNKNOWN'} on ${nfNodeLabel(amf)}`, amf?.state === 'up' ? 'ok' : 'warn'); const upf = nfs.find(n => n.name === 'UPF'); - addLog(` UPF state: ${upf ? upf.state.toUpperCase() : 'UNKNOWN'}`, upf?.state === 'up' ? 'ok' : 'warn'); - addLog('βœ“ Device registry checked β€” see Prometheus for per-device detail', 'ok'); + addLog(` UPF state: ${upf ? upf.state.toUpperCase() : 'UNKNOWN'} on ${nfNodeLabel(upf)}`, upf?.state === 'up' ? 'ok' : 'warn'); + addLog('βœ“ Cluster subscriber path checked β€” see Marvis AI for node-scoped health', 'ok'); } catch(e) { addLog('βœ— Could not reach network status endpoint: ' + e.message, 'err'); } @@ -486,10 +540,12 @@ function clearSessions() { } function backupConfig() { - addLog('β–Έ Exporting configuration for all NFs…', 'run'); - const nfs = ['AMF','SMF','UPF','NRF','AUSF','UDM','UDR','PCF','CHF','SMSF','AAA','MME']; - nfs.forEach((nf, i) => setTimeout(() => addLog(` ${nf}: config exported`, 'ok'), 300 + i*120)); - setTimeout(() => addLog(`βœ“ Backup archive: p5g-config-${new Date().toISOString().slice(0,10)}.tar.gz`, 'ok'), 300 + nfs.length*120 + 200); + addLog('β–Έ Exporting configuration plan for all discovered nodes…', 'run'); + fetchNetworkStatus().then(d => { + const nodes = d.cluster?.nodes || []; + nodes.forEach((node, i) => setTimeout(() => addLog(` ${node.hostname}: profile ${node.role}, services ${node.started_services.join(', ') || 'none'}`, 'ok'), 300 + i*160)); + setTimeout(() => addLog(`βœ“ Backup archive plan ready: p5g-config-${new Date().toISOString().slice(0,10)}.tar.gz`, 'ok'), 300 + nodes.length*160 + 200); + }).catch(e => addLog('βœ— Could not inspect cluster before backup: ' + e.message, 'err')); } function reloadConfig() { diff --git a/config/marvis.env.example b/config/marvis.env.example index bc9e3bb..d809e6b 100644 --- a/config/marvis.env.example +++ b/config/marvis.env.example @@ -4,6 +4,11 @@ MARVIS_PROMETHEUS_URL=http://127.0.0.1:9090 MARVIS_PROMETHEUS_PREFIX=/prometheus MARVIS_ALERTMANAGER_URL=http://127.0.0.1:9093 +MARVIS_PLS_BASE_URL=https://127.0.0.1/core/pls/api/1 +MARVIS_PLS_USERNAME= +MARVIS_PLS_PASSWORD= +MARVIS_PLS_AUTH_BACKEND=local +MARVIS_PLS_VERIFY_TLS=false # AI backend configuration. MARVIS_AI_MODE=rule diff --git a/config/p5g-marvis.service b/config/p5g-marvis.service index 8b2592b..f65d368 100644 --- a/config/p5g-marvis.service +++ b/config/p5g-marvis.service @@ -11,6 +11,11 @@ TimeoutStartSec=0 Environment=MARVIS_PROMETHEUS_URL=http://127.0.0.1:9090 Environment=MARVIS_PROMETHEUS_PREFIX=/prometheus Environment=MARVIS_ALERTMANAGER_URL=http://127.0.0.1:9093 +Environment=MARVIS_PLS_BASE_URL=https://127.0.0.1/core/pls/api/1 +Environment=MARVIS_PLS_USERNAME= +Environment=MARVIS_PLS_PASSWORD= +Environment=MARVIS_PLS_AUTH_BACKEND=local +Environment=MARVIS_PLS_VERIFY_TLS=false Environment=MARVIS_AI_MODE=rule Environment=MARVIS_OPENAI_API_KEY= Environment=MARVIS_OPENAI_BASE_URL=https://api.openai.com @@ -26,6 +31,11 @@ ExecStart=/usr/bin/docker run \ --env MARVIS_PROMETHEUS_URL \ --env MARVIS_PROMETHEUS_PREFIX \ --env MARVIS_ALERTMANAGER_URL \ + --env MARVIS_PLS_BASE_URL \ + --env MARVIS_PLS_USERNAME \ + --env MARVIS_PLS_PASSWORD \ + --env MARVIS_PLS_AUTH_BACKEND \ + --env MARVIS_PLS_VERIFY_TLS \ --env MARVIS_AI_MODE \ --env MARVIS_OPENAI_API_KEY \ --env MARVIS_OPENAI_BASE_URL \