From 9a7ac19cfae63644d357f0710de288ecedd62326 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Fri, 24 Jun 2022 17:52:24 +0200 Subject: [PATCH 01/22] update logo --- .../src/main/resources/kafkatorio-logo.svg | 2564 +++++++++++------ .../src/main/resources/mod-data/thumbnail.png | Bin 10274 -> 8176 bytes 2 files changed, 1710 insertions(+), 854 deletions(-) diff --git a/modules/events-mod/src/main/resources/kafkatorio-logo.svg b/modules/events-mod/src/main/resources/kafkatorio-logo.svg index ae5fec5..77c43d2 100644 --- a/modules/events-mod/src/main/resources/kafkatorio-logo.svg +++ b/modules/events-mod/src/main/resources/kafkatorio-logo.svg @@ -23,7 +23,7 @@ + id="color-profile6421" /> @@ -4840,20 +4840,20 @@ inkscape:pagecheckerboard="true" showgrid="false" inkscape:document-rotation="0" - inkscape:current-layer="layer1" + inkscape:current-layer="layer4" inkscape:document-units="px" - inkscape:cy="183.50283" - inkscape:cx="660.88614" - inkscape:zoom="0.72478445" + inkscape:cy="568.78049" + inkscape:cx="196.09756" + inkscape:zoom="0.5125" inkscape:pageshadow="2" inkscape:pageopacity="0" borderopacity="1.0" bordercolor="#666666" pagecolor="#949494" id="base" - inkscape:window-width="2557" + inkscape:window-width="1277" inkscape:window-height="1528" - inkscape:window-x="0" + inkscape:window-x="1273" inkscape:window-y="0" inkscape:window-maximized="0" fit-margin-top="0" @@ -4875,7 +4875,8 @@ id="layer1" inkscape:groupmode="layer" inkscape:label="Layer 1" - transform="translate(360.42908,-216.74977)"> + transform="translate(360.42908,-216.74977)" + style="display:none"> Kafkatorio + id="tspan7786">Kafkatorio @@ -5636,848 +5637,6 @@ sodipodi:nodetypes="ccccc" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -7260,7 +6419,7 @@ inkscape:groupmode="layer" id="layer2" inkscape:label="cog" - style="display:inline" + style="display:none" transform="translate(360.42908,-216.74977)"> + transform="translate(360.42908,-216.74977)" + style="display:none"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/events-mod/src/main/resources/mod-data/thumbnail.png b/modules/events-mod/src/main/resources/mod-data/thumbnail.png index f816fa546d927c502225e83ddee3a21bc9120df0..693598dbf7c7ca3d25c099b339177c6a4e9f1e97 100644 GIT binary patch literal 8176 zcmX9@WmJ^i7N#VmJ4K|syOBm(L`ph`l{|6aY?*LDE`Fegqsmj>=yK2|mjA!vKE-i9X0y-in zceZ`Q%pk7n0WneEaihL=&jmKpDD)PBCfb#}`P;np-bg7-Oa@=eA(5fqdE3XKL#u+G zpT>ud_bI(zSX&c(r*9@cQHaD@ShtqqPglaz$F@gv>D24|clB?&`~XKDeGjeou%$$y zyOWWdCti)5$PHclW}wh`X^;Z*4&e^2IEmY99W-M!o5q`EcS}b|Nk}5K-lV3@s^X^> z7V24A7R+p@b{Z~hwvw%re`m0!Evl~hiNhEex%<0ke<@9mjFdFWvL?$l8UKd0D4kTu z5pQpA@5WR^f?dzc$v%GcE3|(C8DeQVmgwANr3iZY&}QGIZ=}85`Q4NCPH$EVLUau) zM;mh+4v)G-;`%Ius*%dd%5TEXj~Wg_TAgim^Ca)6s;Er-rAF20$u4$o`^`$)e)CZvTP>`w=l*Rm_ww%$?#g2xY8EHuzm1;KTkxm{%kGDIHQ*(2*^!4>e%>Sya z4CH)yr5u~gidPnzRP*ty;%POaBOB$2O+eE*T%zvEeu*DwbmIw{kl1~NlZ6;#4n{Q1 z)Scbk|9X0GtgNhFsIi=$p7OaHSE`l_zjqm7mz9-8u&ZJWb6Zo8c!l)c*OTjhPRf$o ztFoG!l7enD=tDtC`O3KR^mMh*e4=7ZCyW14wve3~s-N5X3Ehdv&dw$mQrw-HS9=W0 zx1Kqkxj&Q7Zhr+uKrcQp)da_Nhl?xj)79m6o?J*~U7hf{9D7>a{_2p=<(Xf5M~BOF zW3?C`3rkFQ0yX)qTeB5sL3~O>1#+hKj)YVR%SZjo7+$zE`hLs*&bL=*T6|D$1z{=% z_Jes+7du~w{~QJ%1j|L7_FGAMFX6&qb-7%6dV0krC2>hf9AaV_uV1t7?d|=U>yY6w zu5!-M@%It-n343~nh838Q;D&2yU1rWLp6b_*1k7&r4hw{FN=S9e@LNccJ>x*RbO8p zr+`3daq;KHI$1e6gK{J12D9rA`T5zUrE^CsAGVF{xxEiN_qVmP^ZaD=3Zvcg(;k$q zc%^}NOW-bZR8-W-*;y(QdGGP#$1z=)`etTYj~@%Qw6rw0w7eQcKtWek*4W2Ne^`85 z{Nv7m7AzczVp2D0^XHqJo6AP6sAWA6>%!sW_}U{wpvyg2RRv;VKp&c zhMWt}zkmO*u#D)Fxv=wfd(ye?!|XhKO`~@dIIuFbR5dhk1v(B4OZq$Lafyj*jyiH9 zFD@=}M`_eg*GdZ!FEQv6Ipd+mB^w50a&qa)*_qgg9R#dCT8z6n=ff-)H#hmt!wm1= zpKNe_XgAR2`W;c(_PinYPS4TiG#4-Lhnyam86}Ug*RL)-LqkK!@C!+qfGw5Mk;!(w zR95&T&t>b8awX|^IAHg;A%THj(4pAGj24^0fq{~-#J3w|dVJ0fT%gzJ-B)Ly*=Cg| zBqZcW`LHi9FWWmg*$L9PtyndAl&#Lbf}y&@r(| z7#L>C0-EXdt0MUywRV-?FWuSOE3T@ln)=?x&dA81rMksQ-`zj^J#4fy2LUg?wdDn3g&3< z5p?t$R7FlTxZ*ImHkxiN*#-r*%*sCSQK3szGtLQ+iKoK~%A*PU?mS!h1J2aFOAAd7CI@)47-M)B2KH0Ly304Lz%x}@$*&ejt{P?$8evqnx z{pTlmVh`ghYn_LgktoAaZ#8&s(8*$K%kf+Y{UuQVW_)#2#dl{>K%W>!x!{R`tb&5U z4=y8rwapvdXhV)RYJ(!i-#yBa@MQe`HL*lFmWW>gk@F$<(~r(5OewF0NEFdw9686I zky7GUuU>s{9ez|Qt)M_xOY0_} zKwW&#n5bf#|7NzZL;Q$a2hv##!?4`w1&z?>shmc(=dJHY@nEHh+Az>vHYS8FE-%-hkzo}MuwdPW5-t}pvG0HhE#AxVBTw!>?Y3PUt2Q4%N(_}Rdra0n zotQA8;xp^wC=4}pDnLqjdhh;H4Gj(bHZ;V+#`ZLkUP5~G<=eLpekvAo0@NvmhkqPh zMA_Er8yGBl-8WXv*siy%>9H0}&dgM^v3WI{5h68^CzV#Bo@85V0)V>sg;?P=%w{HF zK8!&=H2d@CyD!vAMnCmBi~ic&OjIjz@f*RZzN^Tc5kGDfkq?NLl9QvUsE8KL^u0u# zIQ)E5;+@qe{fCe784}i!NMgVum@h?b2X5|!p8nl@1z(0@6_ohLd^5ir8r>E z3tACu2?+_nF1JC^fX%6l>};Bqp+Y@3H(nsa<(wOcBgy&0@d^8IvlO>jqcOZviEp#@8+L(7;$Hebl*2H z_zy;}+xN@go3$mX_AZ#r_bjLU=nluOCtMJ8VN?XdN49N%4J2X#}r;02D#dD5AWq z%=&n1Hc@DJtr=0I=Cv|FOGZZaea-i}aje2myWd@fmG(ulj!ZPHd`=xZXqLFMS-+tsC|3qbR=UKm9vTcS^}R=Xy)N8H`mD%2#Zju)XPjJIDGy$D zgmQ6l**iG&HdaajOsg39I{!e;t$cYq!%0I$8bJqjf#P$=o}^Gsfd%-Q8VQ4h|rtjX|`8VvX={`BL3{ z<~w(y{`_%WTPoPB3I;%eGsE;<_H>~Q@dPoG9`urz>LvmawG6_lPv zYM0#cOi6Z;Nqj|=ZoYK4YfXpaE%&9>)!*COLwy-MnmRhmuVNW(zdR$8sJw=G7O*nT)>etjfF3O+wxYX$}eF08CP>$Xi{vsNO3(j2>DB_$;# zl565Jco8WqA`;E@aBYZBK)`8zteQdE_j!Zs_*gu; zNm?3*UY*zCXEnm|{{H@3dz7v4&A_D;_2%$PX$g-RYv3os@Wy(0G!?H&=W>5m_5Ab) z7iA9*e&}%|a(ZcKXkfq|=EZCF;dyCcc&l;iq}$5n@jM3?SFcCxY!2}8ZXZw0M1Hvk z7X?d^Q(;(U<-;1Yh&=Nq53zOP8yU=|ADl_CW1@i1&1yxZ31p)k0N zWs8b*57PAHdyAA}IQjUxQK;(wl)C5>ST(CGJ96Ep8}De6!L!)J#f=*Si;Ii1WrMt{ zta4XMiI6)Nf~GpVr4{p0PHrqM3|8!bTgaF z9?wT8OgDKt?Jf0=o#p`9)&aQ(A8nYr4yp~d0iM=IT!r@#dOA|DF_q;pxM|O;SB{cV zQK_k@VC75uA$}~Z0e>j`+f51ubK$!&QEPmyo8*EI%BZTe-WZxXRnN}O4zC}|9(-4R z^7QHA_I6TDjR0Wp((-ajUf#^crJ}ll5xKFY<)HfQu%(?av+7KRi$!7rM%ir3j^NR$ zsFcjiuV5$uZDU8qQlY1Ye!RK2iG2)8ggup&lz~Cp)RY!}1%37P1+DV=(IyNBm8NfE z@(uK66{*YR<>fj9Avd8a9;cAd@M6wl3X8h1>lp9U2bcSv1yI|=<@Y%X!j4SEJ?93X z8R-3-`++-9l{Fu^d1#c?)YKYG9UIJ0_yXnF4~PtLB0j&D zr(j_+ppf!jdcJW1I|7?5EvmJ!9T{j{{ce(&yGj4!doSz$wfvqQid zC(Bt@-v$O)1qI*F6|Dj;fPt)hpsIfV#T!jcO(NBJt7fk|U?|2ac0_v!H z#0Ls(qR}0R%6-R=fq_wxFM;q3E*6W!q4*g$*W_8a(N4~x>-Ijdq8Ylvq&={96mPd@AIiAc`Kf`PUhbdlI|;Do%Cu{9IR8CrCp861rp*!Ntv;Ivcb#QCl*? z5{MVw`8!F0pQomSJZ4KR_mN_gOpMgu%t@ogc^I3~^weTCZuITs4c|7Fj1s30IM~?I z(+p^ntI<-wyZ*=|I#Pe%EOyZ-OM9#6f%;Xpu#KI2Dd62E(md&w*z}O_Mil%PgVRssZf<0BE$8zPdM+;I8$m_M$t0!n=ek9zf9=8F03lzCLbS=r zNpPIn#)Zy3FK^X3B`v=XIrf+g<6%2zZ=#6*Ly3(8&)N&ySy0j~l;2EWrJ|-Dr>`$% zk7g1CLs%+%tb3Of1H=B-GFAplQ&*SDO?-U%Ls`vKlUCnyx58y8qt-J;zakp+?(Xj1 z0+aUOLvEkdLHD1HOMdHPQcFI0g@yChw;k#y2MXjuOge%aHt1>^;OoFIQa?2>$3Oi8 z>Dm5;zXDthD?7XNg>q(QW@>VBCoJ~c;9zFh{NK^a9LdMb2~>Q)fX9D<4Q*PBIrvKN z`QCM$zx({|!&fxu=v_TMGU->Kc|p-Vdr)aSDfeC{m6eD=vh!s3_vTmcMpbWbVW?#X z5Ic3@OD|p!fO6b9I$~#KeG(rZAAI?-$#Z@XSm;|S+r8%r2|j1f>$&u2OleuvU;6$z z$ma%29<)*06L>JJ4%VCx9UYy5nmR2bLluGG27gU7%HaL{#S8o4VpZ|+Kk^ql%JP>7 z_hAeYRnkPY7`2jD5N?2zfu>Z3Lx<9Ln`@_4A^`>P>hH@as%t|UM99{rS^loKI_Rno>xEa5*Fx5%c z%4IQp5^2Ouc`uXq70~6FD?9@b-Xylez`#I<%EYi_cn5I!z0ayHQbrztF>p0MAaK39 zX+tfeDV7BTmPiA>$Z_dg$~CV%dGbU}ZaRnauA)Bek1dqMd=)!$qM9;f5!d0+Y^z`B z`DPQrE$N>wC?pfEii*nhgI5OgdCt|t{+5@Xfas5li+g;M1LenS@;(w2p8xioj*7~Y zM|$d-sQ~7$%gS(L{_8y5Uqy;;HL!4cFbj!@q>WpKB*GTDa?a&`IRlMoTx|yZQ1=Br z4@;@DzdvEFBSZuYg0$b?pPV5PCyO|leCF)^l|tg;nL+z2qpcpIbP&TOKWZY&(0XcI z`35Q$Tw>$%5KUB^N;e)Z_o_ZbhDt_8>9g(B0+#lm4K13zq5!u?rxkGV@k?CC-@Tv{ ziw7-OdQ=YmMk9tuN0|)4itT_l58paDIFOQ)+coWPZg1-WtHKFgtm#D{-lGUrcSVNv z*Ft#JW%Yv?Lq{2pA?@*j7LeYre$>|3>*&|f-tMLwelxnW=XQi!{R@CWknQgLL?J;z z5GBEW-4PdW{O?GbT=iyIgAJr6;%^|37@IMM1i<}fnbS}~)uXntSBbbfiMGaj$(DA*XiMimq9QmD$GZ-{(6`Gt?%Vg<}j=(auXsgaIpkWcrP|GIf? zd;}l=QUN^@Ve*@4ES57~yHI}C(e%cR8#C=eBaQ`cr>16Uzzkr4xU8z;HN}6&p_{h= z60KYWytQEH$xg5;rzV-ay!^rXc#Xl4_>F55^8US`;#{T^;BAXHTZvj?9rlQDjyA7l zdp2m^DL0H8N<~#wFAmqoc@VUi7GXtdLT(d+N89sB-L^`lBL;?swjyIGteO^W{)ymD z&1J^CsUO`?T3GlmCQsTg9(v~|eOZ0WW4tm_6*VXt0AMv^zQsR? zj@VcnfNLODueBd+S&>^59`m@@Qg3af^mH-ycAoJ^CfqSLf|6kkdwbR-xufinK2P3v z^2y|+SW!S3t^S*T)Az336%dkiK9KC>`v0t#s_^$;tP(I!MG8nO4bz6U@x;)kuFekzK_4{|e{O7gtn!?$`N?H*|ujM{UaE@wFFqh}Y0TqNgI+dg(ogka@ z!e5rLYGPqwT?4CRgGCEu+2S5E6U;;Y*wt_GFejQH+oh=_2yk+ijZ=<2bvH|l^q6Zm z8^v@ebh(UZ*BfcbZGQw~{R16EsOEUelwNz=(TWFkm=XQm zv$OfITU%EeffEodaoH=IqCT4T{8a0RqH~Rmf@`Gx zNvFcc0M`(nK6H(-fpnEv{)7oaO^2E0#}J&pd(`$75FXOO8~9Wtw6v3CH9rv`uvkS! zKR9xnsYm8S`kt@0tQMr1@qm*`3qIcBgDXJ|NKJl3qGP^>t{u-Ax2@HHOEZ+4&H56k znRLWotdj`t1=pe}hAT*yKf`?>(qdFc2?zyv4UlTEwY&MF&&mVtjCEE=+UxWQW<8#9 zphuZZ#L#|O!5M!6Q04U?;L;dC0p2< zIYZ__lorPkTt8I*ODkdJk=rW)_f z%j;8~m(PCH1iavzy=Zm517OdZq4nV5{3?OI>z{MnWUH*_FdBc3_S?+x+tjEhobf+k zkNNPT^|Jy8j2eyo_L!N**opxVwN{4;J=X$!hP&(?9kuMfz9_#Z`Jhlte7gP0v25QF z*un*F=6--^GCPxdB$)%**V^@l!IhI++^Tu!R>~B&by%1fDN|5jpay_v#xQ zwfqz^s);Mn?YxXCjg(iUE|RPMRx_oVuc64E!9&JTZjo`4+b942(LaSWV^#$c{(~~O Ptw+;T)m3?S*E;%tZ2&f# literal 10274 zcmXY%2RPJ!9LMj@;p}tv$QIdTryONhPT4Dj>=Cj#dqgB;mL19}GufMD(-1O}otf=_ zzu)~|508h(!`<)q{d_*}_xtmU!Q54&AY~?nKp+$v>bLIUz9s^nNFoRX+B!n%jQa-e zp{${cz!8A3j>dgY;-+rof%`EU_=Nt3&BAa$Wb#xo^t|tC>*;Oz*aqV5?JZ*O;^^_v z(#=N1_3@+3O$BBMgae{+OG($~`-YjXDV-k64QA&wdS+|CkCDzZ2YJj!U`5J$o>NVT+6w{-_-rJsfq_JL8&dK8y5@n0J;~H z(-JlcM~g4CZyXiutZ8mJs+i@wXw)d$wBp~)K=Ai__W^2NC;onOdj;-R=(CDmqFWkB zG*0gac^~kgERHEid6Km8SKYhs;U9x9&d*N%)%_gfrCD9*S)ud$!&zqVXU2c4>EDb$ z$+!151F6C$ZyI@=t0 z^5mQWMgTxkQVQ@s*KKX$+F>vTAaOs_!3Ur|a**z&?s+mDEx=bxkF(Xc70t)1g6CzL zAXa?Oy!KZo-WpWYwz&*-e6MkwDmQcVZ66<;51!R{OCfyG@)UrKlR?C^-tAZD6F&4V zNgtKa#gbo(U4Rd#7uN;Pq5h-kVP(^fp&8`BPdXY|2#Fhfcj6K70uwJbHywyRig za_@$X*SO2u+FDxwy+0%QYE+kEb%J1KA3aw_3!jF(l;s782u>(LvTT(Iz~6ngqB3j9 zJ6yLcJV?@g2qk%b;qr|zCVt{g@_4>Mg?W?Teo_Gz0J8f)HeKD3>d(rA@dVv?Ri^&p zuh+rnN6W8+lSQ$k5yPJd3}~|AZp@X{&G@aA)xE!%4Zb-0Hyhkic&p(f0P6APbc4t8 z7ittWK$~LFk6LHkhgNUP#rerr%f&L=?=DcEhbZ`}t}KY}fsg6KVMFX34s@`(3izRU_wPr30`zI=|#S?cQ_A zyKk9uHgO_$%P8_5to|x(NYGD%REM1(pco29H zFU*dpYxRkl*fwRKfEOBJKVGbpuNse#3qC*fm8_WNXAM!7l6|ZB2iH~J+|4%hgrruM z1seL2A&>nFZ3js7SYqOG&9^YtVsc7CEyCJLgg|IV<=s*5zWswv3dviY@6x}MRaZ1*Qf z=O0{)xB>>MSTFkT#Ps#=Lhn+#lSQdIvGz_kvIRl|sq6zi!gkfA;kq2R~2iMef?p zs$l#N|DRCr$Qcu`rc%8FiH@IS66)_RbjP#ExHy$$ExB{h24;xZQQc0Lu<1{}l5|V3 zUr2@B5gbacq8hm%rwiRZ09yBK0C@egRk`l0I~Y1HJADNzg(9^r$iL&SS)A$)PmHZh zT7lEdC*R%Qg8_xch9F^MHxOR^j;N(JFW%k_jxqJ5CU$WF%T&#;LJ4WV8X5IQo&5D- zwGTgBCKrAp-fwEt_{ey}#m05`*nv!odGj^T8PM4-&gyj6#2hDqMr?HOZu0pu*Pnjl z(Zms_(E<(Jk#xA5^#~kiTYLqd08GzC43EywO_qLPLW*?XU!Kfp;A%g(fL_)++|@pU zX3DtYUZ;mfPUON<(dNVXr=mWggm(f6R>e-q0z<*3M)xe=PbV~{RWPV0exQrc2krdD zx$ftaAI^1svp*Z_C{Z+?fd@u7eS<@plKn=&;a|($Q>3rK%uTB6d&4)WsOMJzG|M)MOxB7r3tkby6Lt$(MzEBv%4l~ zN>^d`^VK1RU2?}aUTIoVcEdSJVMJHRC9l?&ghRhmY*gkx1iRQFtWrm@T=^zL74#tzNLMzu)QwYk<9i^@mDy((f0B ziJ1(vYU&=H(bU|kamdiz?{K$d8Wjxtp1eW@G_;3?HPI_I1%S!BIUOW?IFD>;COEVXLhHDV-QbzznTr(iThpt^=*fZ>g-&k*CNFkz^_K z(HraAl$vEAc7Kf!u_-Cs{o!mjOedPImOkzY z8?tvA_|HxLBt|t36Qz1@5gK;^CWCJLKh-pPUtRY zJ?rv{HY9^-ThJ3l(;9Q>3Jz%`x!~yw`FtSrdU#SBOe%sHC%KT*lnN=Yn1dO*I(fYh# zOT|qY&y{}M%3Qj3^+Rm~nqYwoYOcK@zTVWFdjA?&~s4wJN%j9;N}d9c zTSxxU0Ubo0UiI!}?~xoGQmJKiFei=bsxLc%j_HMv8o3(d2R73*Hf5i)c@M%np(Go5|YsiAzePX3`5bAJt2$QhxP72 zzwEguFhR(;Fz^x(8giU~Zn*a3oo_`;$9Hrz)i#{C>z@Lb>CvQBg?k^zf@I^I>@+=5 z9ep>iXeC1F`9I;3{ZM{)amTfEG0XOF5}NS%1n>A2L+I!jkOfl56b(?9OW zYslA+BAW(JMeyG}I;Q}DS3HTbEx#5FH5r2J!WF^F6qPN>* znS&00MX?7RDka&W?kD|u=h$}gP{@mAQf3B&KizI z@?7P4aK#r}Us*#z0-_nJYbr#;s>S85dGy;y(ewt7W~=<+5x^Ca(HeD0K?|SFhUHiO zn{qvcjVWr$_8dBgCVuX_cc<%SFF!;1J_9YpLyu!p5T3qSfCs(@^pljA94EuSFG7m{+_<@VudQUl!;B zA07!a&6hXW_-To~C95qZ><;(hl$D9Pmpvf`ee4AwwMCT%j-lPd*B*}n?#+r4Cf$Js zx-mvdm|qdB7`a=LLvA7+yP>qsGdBkdXZez82DKG?)vcsfj56Tl|1<1uBN-ouTD zUr`~tsf)zkQY^Z4=#PKg&{Q-G%8!dQZ+lKA-K~-d-O{dRr*(gBUl~`NFr^W`IVe)-rQ?|TTGThInnyC?sC*PDl;s>m;@a6E8*mT zW6Dud#(EOmG_tt=7v}cY;jdg}A>eOtLg09a-~}mulz5*nrx+Su1FbdReoBslv>`R| zy}zm!PN3O4KYFw6PC=V+4MZ@Q?GCX*yd|yJXNJOpYU!+{)AdWl5@it1&9=-&7s(sv zhYJac=V^)hWyZJ(6}vQ9E%D8StbW6m<-OyvwLopwDm`lsIKuv2wg&D%6s4Z1?3`}6 z|1+3$Q>ET!MYpRZ;CQCWFKHRs5ymcV!^0HD+!nS={hw(>9r{1UCQ&tr2T)Oa-KNXy zG+g(p-?zw?dP^j6M}R;>rz|w=F~a;l` zZzt?*pF+r>A7hEx1OB!Vvo8jhR{{(_nPI0&P}u3^PEg>2cYDI%uF=GC7GlhJ>&I~6 zfxl;&NX9Xe#Cv@us5oY%+Lh_JsbppBI^6T>UkvC~%Y((HJ7WNAtgape(0shbdAHDO zQ{ni_-$3)}z7hF{cV6}n|6q^Eu$~t4ylN3JhLaGuJ7qKNo0k>-@&mpNS=9{o9w=^i z(r7De!#T@9Pb|f9&h?0-s&>6|@>=8y9I@LxoVLv0+Svj0Yh2T|rF+`D>DYyZ5)hFy zq7m75tjwVjW<1RMq6Z({iec!|&L04o7fsOs2>rSO12-%vN$c)9S?Uw)i&cTto?PRf zZ51G{<#5uN*eDF`7}2MAE~QG)2c>|-V|CIdVRS)uFsrz!%r~S;~%4#fB%y%S%n{@=KH7bc9s2B zJ9)#n+F_2N)VzP5H4;kjclvRS^=k!afx^@Q4eMA>2rEUUifSCAxK$@&X{bavabZ8O zZ9|479I^suCe=9|qLJi%#o<{i;N@$fb=A@ECL-}}t6(R^@2bd8C!391%_rqH!ORs? zm!#FK0eE(a24Q@hq{mow$?k-ki5!dy#m+pp+^_pYeU^r+*>Qw5(Ui$a^TshtIpD17 zySRu)jwH}|NH@RyO=$8%3&v>FqMYF3t2j2|h}K78U@Vez?msMvWZY}z=A)+pc-T(d zqce~G%8Z$&O!<<3rZ-CKRxz^qA{$`v-W9(P=~syWNa;@1FWs^4ZfH7CQ$lL8CnXsC zyFysIiIh{XgE~;{!rj6j?S?!4INmX^LSSDUHC(1aIbxJ>E$NLd9cz?KUcBx76xT}! z?f9(jcYW>aj<0BZeAu*elGB%Rzt>Vw2R3b0W|E8TbI!pF)ED1KKe=mvGNm!jNb$*F z0pBpUej3qxcu3|CbkPJFpMD1RFtFA%T;8h|gyu_@l`WJwiF2u zhdx}Qk9@-PVZIkkg+|W9|606FT`}fE+h#LRC4;U)+*QN%ln6$oDlQhrbZ5p0*M!9S zgD;F2M~qkWOUa7Oh}vjuRjolwT6HcUhh$=t0k2M489|ZnVz0$bl4(Y~RDFFqF+cyd zpG~jx%NPEg-#Ukxbo@8e0>6L(qxs-Hdf0a?k5mS-yyvOt42&$STu0|?zW_+wno5NT z2Uk1_Gmawr(=oW>DwHvxlSSgFo>a^tL^6&VPJYItWAePOZQs$FM{QH6MV-4j$rA`l z-`tHMGaF0H0(i$GW++S&B}XaB=$m}WY(2hsirS^BA!M41bbH))c6WJ9&->0|BBy7B z(clZ+D)|GyzZ^pS4yjvTvR7h>=rK-DS-AD9J{4k=!o?k3$L zrbRq5LbraF1?O-SPfV?kI%-gMttN|Jbh5-vOrt7Kd4s(mq7xShjCaM;B8xS0K362| z^$S*f+UXZGrlS_(2@ZZXzw-5YdGJ}G{;%QR3hrS}gvR+i8r5y+r2o_4+WSgkaQHfO zv9IO|4-ap77j}C3Ppp8%Qi%gz6NHCH0FcanpNvibcvR!5ZjxJ!Qc35iQ)6kyqP<{? zey>2{q~^NO|H@l}v)2NpiqaP$zoN_R1RR-%dG3($7&a;!12D6+L$ZLtzQMzZu|sAy zV0TbvR;(+(^F4xa9E>G0Kg^iyO$tzR8R!F`cK(mModU!(x8d+T2v0ntmZyhGwU^_i zKcXLvzue1|vtK(oS;J#JsGi=H26GM;DmJ{4Nlk`w@`J(?m`0#Dg)r* zgw|@P|0YI#VP8cCm6!WNC&gW$G0FR54ZpOMBw z#p(Rxxt@=%@g?O16p9PA8eCle;XQ6f?HB;&9*3VGJ(uDlM?b+koFD`VP>%XkLx2R1&};Y zeD-D7L~klXt6zF)Z?FTOmUpfe!vo_nXxL>>aWe3aeqGBUR1)l~VGL32`d@=p8D{Bs zf>cT%%CKt@^5Wu>DKzrPm*^b%kezUx*~eLzFq?cB$R0ZbeUy0@LC#;AhpjcR&c`g$ zK->^kKdTHSkquWS1s&>0Fl+vs7!!X5-S+&byXbn0BG~FZ%$s}v=Wf7e#Hzf2KF6F; zX#|%J;UL%5H%7azG%%3*K$1HqH2y_-l4`tX4%#UX#HcdN3{h!)z}hN- z?T$J=^L0J-RZ_~)Bc#e3fe40tiXd#W1H}lR?Jtf4YhmQu(7d(_YTY-x+u`%nq`ej# zCcBd+u1))sCUM$5I@tD3&YO|sM;FiG7V24GV(nqb$xVfj5H+l{z-24%T2b^FO%N3~ zk0j3x^g7AnIX{FBSCE&{g-#$soIoBDXQ?VhVmIis-No=AU|Xg3LolG&1O7s}l?T8@ zCYLonV!nC~-2z?AC4}!>2r-+_98@}}mw!M-aCj#q)=tS%kQP&8fj|3Jj?M&D`oJ!X zn!1~P_9sAgym-4IgWToYX}DMhu39Y(q~Q09koUk@Qo26Rte1$Gm(8au!<`?QPB61# zRae&lh}k%&TvciB05B3`Dwz2s;$2PBw}mP26bozXdV?z1{0}yj8Eo7d2b6eumD#+NL zBBbhqUaPf%R!Av*rRRE~Pcjl*pE@b*3kYQC5OZ^I&?b`Rn0FWJ^da&n0jE9J_WSJT z95yn-miCrlC>Dce&XJU#G=ABm)0$~fLI=vb6yo=GW+{%zZd*%{s{4~65QGIVMoBP(VYHie zj{;*GOgFVNeNH*A5Wwg9c6oBKTZA=O;AKEahyB2{^3d@@F3h_XG)xwxedcBGKO?aF z?5lQhFw%W#yvSZXG5e!LkmBe7Ai$}WDrEHDIwsNagEB3b0EDjcYoBwWnAkuQ-cH-b z1^Cf%(9YoY<+y3x&ej&SzMQk(K)vfGv6vqL*eK*;Hm6+udN&zvf9AJ`KXCkSK52e)7CxHX~#8!z-%=r=+Op4(vQRArr8 z=vyR-$+z+C&Wc3DgI_`59zZ zeGMI((HEz$mrQe~+8KzKx)m_xBO*ptLKpH}EPOrme_ItV5&1OF0NQ2) zGx#?O&Y#jVS+|H^%xi>2@d!6lJ$DkE9Ihm6ZO0B+35(g&{XDC@)jt66Z5cC5c-Owu zTFl_KYa+9e@@Mb2_C`9EP?B-PZQQTZfWU0gBF)B4J7|6+^db zi9)}m&4i?}_M=WgI?otL)mv#H|7ux_--OIFfx0m&JYp_Z*%$d>;#gNPS5_&u`M}qdHj-)Wm zU@1W)4>@ATu?EYaKvWw+8E&rc!Bmw!I}EO3+Mb4lsa|2z9d9qhOOuCfcO66eO6}z7 zkVug<0?3M`p@I)~UxI>5z3inHv)K`NP`-T+`=-wBjStPj@oFmeL zI+PWLEf{K9C5&FY2awnxfaPNFkM*Q2MSP_r-5*vE;IN@7AIghDN$D`@=MUAY%{WM^yE^{6%f2a~;54Vsy0IU{aVpr()Xz`ALi;M^Nno5MZ3%t0w01UI^^#=%I5fFx|Y2Hv9D_teF`ihC2KM2Iv zPQtO{IRr)$zd{Me@1+Fqr#TQ>-L{*#HC3wf6a$*kd42alch%v6F4P)O-#v%sW3+OBnVzYG>O7i zvEt&JAh);YRC7*PrldFkTAx(Wd8R*%B~TwX1Jz~PK-AZSdmhDDEIqo}^^29;@!mvm z9IJ-G;vFFa`EU18#kLp+j7Hvq15eD(pURY4Hgyzlph2+d>PHQf4HxZ{Q7hx{aM*jc zhM?@LZ$u89KbAkJD0n;UL6FDiA5UNIj9&V1?;Ua5$}j+iqOIjEJyiyV5N8WHBmV-p zDWkxk$9GoVGrIg?H-7+!;)86W7X{$}@Z2LpxC4zoUWRMrv7{S6U7l0=SV}3)aZDik z>>|pp(l`nCAPmlJQM>2E;jB#6I}eiV$aTu{o>$o0>oXgtjW+pI=|_a89LQ>*HlC#v zyaN!@$Q#U@S0zK}^4Xhtl*@)ys*O}USL{$u+iYAfHXub5@uQmMsaFWu~$defzym9zHp!Kl}o k{>Pw$Z~JVC`-kk%93Fwmn28qL^L!8umAkjfl`TU5164+adH?_b From f2ae951b1dba8ae900ae8d2c55ca77d526ab73c2 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 12:30:08 +0200 Subject: [PATCH 02/22] remove kafka-connect subproject (unused) --- modules/infra-kafka-connect/build.gradle.kts | 116 ---------------- .../infra-kafka-connect/src/main/docker/.env | 4 - .../src/main/docker/.gitignore | 1 - .../src/main/docker/docker-compose.yml | 55 -------- .../src/main/docker/kafka-connect.Dockerfile | 19 --- .../kafkatorio/kafkaconnect/Resources.kt | 10 -- .../kafkaconnect/kafkaConnectAdmin.kt | 92 ------------- .../kafkaconnect/websocketClient.kt | 41 ------ .../configs/ckc-websocket-lua-objects.json5 | 130 ------------------ .../configs/websocket-async-lua-objects.json5 | 84 ----------- settings.gradle.kts | 1 - 11 files changed, 553 deletions(-) delete mode 100644 modules/infra-kafka-connect/build.gradle.kts delete mode 100644 modules/infra-kafka-connect/src/main/docker/.env delete mode 100644 modules/infra-kafka-connect/src/main/docker/.gitignore delete mode 100644 modules/infra-kafka-connect/src/main/docker/docker-compose.yml delete mode 100644 modules/infra-kafka-connect/src/main/docker/kafka-connect.Dockerfile delete mode 100644 modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/Resources.kt delete mode 100644 modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt delete mode 100644 modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt delete mode 100644 modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 delete mode 100644 modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5 diff --git a/modules/infra-kafka-connect/build.gradle.kts b/modules/infra-kafka-connect/build.gradle.kts deleted file mode 100644 index 69b2970..0000000 --- a/modules/infra-kafka-connect/build.gradle.kts +++ /dev/null @@ -1,116 +0,0 @@ -import dev.adamko.kafkatorio.gradle.asConsumer -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile -import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs - -plugins { - dev.adamko.kafkatorio.infra.`docker-compose` - dev.adamko.kafkatorio.lang.`kotlin-jvm` -} - -val dockerSrcDir: Directory by extra(layout.projectDirectory.dir("src/main/docker")) -val connectorJarDir: Directory = dockerSrcDir.dir("camel-kafka-connectors") - -val camelConnectorDependencies: Configuration by configurations.creating { asConsumer() } - -fun DependencyHandlerScope.kafkaConnector( - name: String, - group: String = "org.apache.camel.kafkaconnector", - version: String = libs.versions.camel.kafkaConnectors.get(), - classifier: String = "package", - extension: String = "tar.gz", -) { - camelConnectorDependencies(group, name, version, ext = extension, classifier = classifier) -} - -dependencies { - implementation(platform(projects.modules.versionsPlatform)) - - kotlin("reflect") - - kafkaConnector("camel-ahc-ws-kafka-connector") - kafkaConnector("camel-websocket-kafka-connector") - - implementation("com.github.adamko-dev:json5-kotlin:2.0.3") - - implementation(libs.bundles.http4k) - - implementation(libs.kotlinx.serialization.core) - implementation(libs.kotlinx.serialization.json) -} - -// disabled for now... -val projectEnabled = false - -val dockerBuildKafkaConnect by tasks.registering(Exec::class) { - enabled = projectEnabled - - group = "docker-compose" - dependsOn(downloadCamelConnectors, tasks.dockerEnv) - - logging.captureStandardOutput(LogLevel.LIFECYCLE) - - inputs.dir(dockerSrcDir) - - workingDir = dockerSrcDir.asFile - commandLine = parseSpaceSeparatedArgs(""" docker-compose build connect """) -} -//tasks.assemble { dependsOn(dockerBuildKafkaConnect) } - - -val downloadCamelConnectors by tasks.registering(Sync::class) { - enabled = projectEnabled - - group = project.name - description = "Retrieve Camel Kafka Connectors from Maven, in preparation for the Docker build" - - dependsOn(camelConnectorDependencies) - - from( - provider { camelConnectorDependencies } - .map { eventsSchema -> - eventsSchema.incoming - .artifacts - .artifactFiles - } - ) - into(temporaryDir) - - doLast { - sync { - fileTree(temporaryDir) - .asFileTree - .forEach { zip -> - from(tarTree(zip)) - } - into(connectorJarDir) - } - } -} - -tasks.dockerUp { - enabled = projectEnabled - - dependsOn(":modules:infra-kafka-cluster:dockerUp") - dependsOn(downloadCamelConnectors) -} - -tasks.dockerEnv { - enabled = projectEnabled - - properties( - "CONNECTOR_ID" to "kafkatorio-connect", - "KAFKA_CONNECT_VERSION" to libs.versions.confluent.kafkaConnect.get(), - ) -} - -idea { - module { - excludeDirs.add(file("src/main/docker/camel-kafka-connectors")) - } -} - -tasks.withType().configureEach { - kotlinOptions.freeCompilerArgs += listOf( - "-opt-in=kotlinx.serialization.ExperimentalSerializationApi", - ) -} diff --git a/modules/infra-kafka-connect/src/main/docker/.env b/modules/infra-kafka-connect/src/main/docker/.env deleted file mode 100644 index 3af8d5f..0000000 --- a/modules/infra-kafka-connect/src/main/docker/.env +++ /dev/null @@ -1,4 +0,0 @@ -# Do not edit manually. This file was created with task 'dockerEnv' -COMPOSE_PROJECT_NAME=kafkatorio -CONNECTOR_ID=kafkatorio-connect -KAFKA_CONNECT_VERSION=7.0.1 diff --git a/modules/infra-kafka-connect/src/main/docker/.gitignore b/modules/infra-kafka-connect/src/main/docker/.gitignore deleted file mode 100644 index 664df74..0000000 --- a/modules/infra-kafka-connect/src/main/docker/.gitignore +++ /dev/null @@ -1 +0,0 @@ -camel-kafka-connectors/ diff --git a/modules/infra-kafka-connect/src/main/docker/docker-compose.yml b/modules/infra-kafka-connect/src/main/docker/docker-compose.yml deleted file mode 100644 index 278ba19..0000000 --- a/modules/infra-kafka-connect/src/main/docker/docker-compose.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: "3.9" - -services: - - connect: - image: adamko-dev/kafkatorio-kafka-connect:${KAFKA_CONNECT_VERSION} - hostname: ${CONNECTOR_ID} - container_name: ${CONNECTOR_ID} - build: - context: "." - dockerfile: kafka-connect.Dockerfile - args: - KAFKA_CONNECT_VERSION: ${KAFKA_CONNECT_VERSION} -# FILE_PULSE_VERSION: ${CONNECTOR_FILE_PULSE_VERSION} -# CAMEL_KC_VERSION: ${CONNECTOR_CAMEL_KAFKA_VERSION} -# CAMEL_CONNECTOR: ${CAMEL_FILE_CONNECTOR} - ports: - - "8083:8083" - - "8095:8095" # ws-async (client?) - - "9292:9292" # websocket server(?) -# volumes: -# - "./connect/input:/warehouse/input" -# - "./connect/components:/usr/share/confluent-hub-components" - -# volumes: -# - "./camel-kafka-connectors:/usr/share/java" - networks: - - "kafkatorio-nw" - environment: - CONNECT_BOOTSTRAP_SERVERS: "kafka:29092" -# CONNECT_REST_ADVERTISED_HOST_NAME: ${CONNECTOR_ID} - CONNECT_REST_ADVERTISED_HOST_NAME: connect -# CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://${SCHEMA_REGISTRY}:8081" -# CONNECT_REST_PORT: 28083 - CONNECT_REST_PORT: 8083 - CONNECT_GROUP_ID: ${CONNECTOR_ID} - CONNECT_CONFIG_STORAGE_TOPIC: ${CONNECTOR_ID}-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: ${CONNECTOR_ID}-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: ${CONNECTOR_ID}-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" -# CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" - CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/camel-kafka-connectors" - CONNECT_LOG4J_ROOT_LOGLEVEL: INFO - CONNECT_LOG4J_LOGGERS: "org.apache.zookeeper=INFO,org.I0Itec.zkclient=ERROR,org.reflections=ERROR" -# CONNECT_LISTENERS: "http://localhost:28083" - -networks: - kafkatorio-nw: - name: "kafkatorio-nw" - external: true diff --git a/modules/infra-kafka-connect/src/main/docker/kafka-connect.Dockerfile b/modules/infra-kafka-connect/src/main/docker/kafka-connect.Dockerfile deleted file mode 100644 index 885c292..0000000 --- a/modules/infra-kafka-connect/src/main/docker/kafka-connect.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -ARG KAFKA_CONNECT_VERSION - - -FROM confluentinc/cp-kafka-connect-base:${KAFKA_CONNECT_VERSION} - -ENV CONNECT_VERSION=${KAFKA_CONNECT_VERSION} - -USER 0:0 - -COPY ./camel-kafka-connectors /usr/share/camel-kafka-connectors - -#WORKDIR /usr/share/java/ -# -##ENV CAMEL_KC_VERSION=0.9.0 -##ENV CAMEL_CONNECTOR=camel-file-kafka-connector -# -#ENV URL=https://repo.maven.apache.org/maven2/org/apache/camel/kafkaconnector/camel-file-kafka-connector/0.9.0/camel-file-kafka-connector-0.9.0-package.tar.gz -# -#RUN curl -L $URL | tar -xzvf - --strip=0 $CAMEL_CONNECTOR diff --git a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/Resources.kt b/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/Resources.kt deleted file mode 100644 index 0e6406c..0000000 --- a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/Resources.kt +++ /dev/null @@ -1,10 +0,0 @@ -package dev.adamko.kafkatorio.kafkaconnect - -import java.io.File - -object Resources { - fun loadResource(name: String): File = - File(requireNotNull(Resources::class.java.getResource(name)?.file) { - "could not load resource '$name'" - }) -} diff --git a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt b/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt deleted file mode 100644 index e8eccfb..0000000 --- a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt +++ /dev/null @@ -1,92 +0,0 @@ -package dev.adamko.kafkatorio.kafkaconnect - -import at.syntaxerror.json5.Json5Module -import kotlinx.serialization.decodeFromString -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.jsonPrimitive -import org.http4k.client.OkHttp -import org.http4k.core.Method.GET -import org.http4k.core.Method.PUT -import org.http4k.core.Request -import org.http4k.core.Uri - -val jsonMapper = Json { - prettyPrint = true - prettyPrintIndent = " " -} - -val j5 = Json5Module() - -fun main() { - - val connectorHostUrl = Uri.of("http://localhost:8083") - - val configFile = Resources.loadResource("/configs/ckc-websocket-lua-objects.json5") - - val jsonObject: JsonObject = j5.decodeObject(configFile.readText()) - println("json5 connector config: $jsonObject") - val configJson = jsonMapper.encodeToString(jsonObject) - println("json connector config: $configJson") - - val connectorName = jsonObject["name"]!!.jsonPrimitive.content - - val request: Request = - Request(PUT, connectorHostUrl.path("/connectors/$connectorName/config")) - .header("Content-Type", "application/json") - .header("Accept", "application/json") -// .header("Accept-Encoding", "identity") - .body(configJson) - - val client = OkHttp() - - // check the status of KC - val statusResponse = client(Request(GET, connectorHostUrl)) - println(statusResponse.toMessage()) - - val response = client(request) - - println(response.toMessage()) - println( - jsonMapper.encodeToString( - jsonMapper.decodeFromString(response.body.toString()) - ) - ) -} - -//val updateConnectors by tasks.registering { -// group = project.name -// -// val configFile = layout.projectDirectory.file("configs/websocket-async-lua-objects.json5") -// inputs.file(configFile) -// -// doLast { -// val fileText = resources.text.fromFile(configFile) -// -// val jsonObject: JsonObject = j5.decodeObject(fileText.asString()) -// -// logger.lifecycle("out: $jsonObject") -// val configJson = jsonMapper.encodeToString(jsonObject) -// logger.lifecycle("configJson: $configJson") -// -// val request = Request( -// PUT, -// Uri.of("http://localhost:8083").path("/connectors/kafkatorio-ws-connector/config") -// ).header("Content-Type", "application/json") -// .body(configJson) -// -// -// val client = OkHttp() -// val response = client(request) -// -// println(response.toMessage()) -// println( -// jsonMapper.encodeToString( -// jsonMapper.decodeFromString(response.body.toString()) -// ) -// ) -// } -// -//} diff --git a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt b/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt deleted file mode 100644 index 1763679..0000000 --- a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt +++ /dev/null @@ -1,41 +0,0 @@ -package dev.adamko.kafkatorio.kafkaconnect - -import java.time.Duration -import org.http4k.client.WebsocketClient -import org.http4k.core.Uri -import org.http4k.websocket.WsMessage - -fun main() { - - val client = WebsocketClient.blocking( - uri = Uri.of("ws://localhost:9073/ws"), -// autoReconnection = true, -// timeout = Duration.ofSeconds(30) - ) -// client.send(WsMessage("server sent on connection")) - - client.received() - .take(100) - .forEachIndexed { index, wsMessage -> - println("[$index] ${wsMessage.bodyString()}") - } - - -// val nonBlockingClient = WebsocketClient.nonBlocking( -// uri = Uri.of("ws://localhost:9097/lua-objects"), -// onError = { it.printStackTrace() }, -// onConnect = { ws -> -// println("onConnect, ws.upgradeRequest ${ws.upgradeRequest}") -//// ws.send(WsMessage("client sent on connection")) -// } -// ) -// -// nonBlockingClient.onMessage { msg -> print(msg.bodyString()) } -// nonBlockingClient.onClose { -// println("non-blocking client closing") -// } -//// nonBlockingClient.onError { it.printStackTrace() } -// -// Thread.sleep(Duration.ofMinutes(1).toMillis()) - -} diff --git a/modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 b/modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 deleted file mode 100644 index c96e8c8..0000000 --- a/modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 +++ /dev/null @@ -1,130 +0,0 @@ -{ - "name": "ckc-websocket-lua-objects", - "connector.class": "org.apache.camel.kafkaconnector.websocket.CamelWebsocketSinkConnector", - "topics.regex": "^Lua.*$", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.storage.StringConverter", - // The hostname. The default value is 0.0.0.0. Setting this option on the component will use the - // component configured value as default. - "camel.sink.path.host": "0.0.0.0", -// "camel.sink.path.host": "host.docker.internal", - // The port number. The default value is 9292. Setting this option on the component will use the - // component configured value as default. - "camel.sink.path.port": "9292", - // Name of the websocket channel to use - "camel.sink.path.resourceUri": "lua-objects", - "camel.sink.endpoint.sendToAll": "true", - "camel.component.websocket.host":"0.0.0.0", - "camel.component.websocket.port": "8084", - // Can be used to set the size in bytes that the websocket created by the websocketServlet may - // be accept before closing. (Default is -1 - or unlimited) - // "camel.sink.endpoint.maxBinaryMessageSize": -1, - - // Whether the producer should be started lazy (on the first message). By starting lazy you can - // use this to allow CamelContext and routes to startup in situations where a producer may otherwise - // fail during starting and cause the route to fail being started. By deferring this startup to be - // lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. - // Beware that when the first message is processed then creating and starting the producer may take a little - // time and prolong the total processing time of the processing. - // "camel.sink.endpoint.lazyStartProducer": false, - - // Timeout in millis when sending to a websocket channel. The default timeout is 30000 (30 seconds). - // "camel.sink.endpoint.sendTimeout": 30000, - - // To send to all websocket subscribers. Can be used to configure on endpoint level, instead of - // having to use the WebsocketConstants.SEND_TO_ALL header on the message. - // "camel.sink.endpoint.sendToAll": , - - // Set the buffer size of the websocketServlet, which is also the max frame byte size (default - // 8192) - // "camel.sink.endpoint.bufferSize": 8192, - - // Set the time in ms that the websocket created by the websocketServlet may be idle before - // closing. (default is 300000) - // "camel.sink.endpoint.maxIdleTime": 300000, - - // Can be used to set the size in characters that the websocket created by the websocketServlet - // may be accept before closing. - // "camel.sink.endpoint.maxTextMessageSize": , - - // Can be used to set the minimum protocol version accepted for the websocketServlet. - // (Default 13 - the RFC6455 version) - // "camel.sink.endpoint.minVersion": 13, - - // The CORS allowed origins. Use to allow all. - // "camel.sink.endpoint.allowedOrigins": , - - // Whether to enable CORS - // "camel.sink.endpoint.crossOriginFilterOn": false, - - // Context path for filtering CORS - // "camel.sink.endpoint.filterPath": , - - // If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX - // support for more details. - // "camel.sink.endpoint.enableJmx": false, - - // To configure security using SSLContextParameters - // "camel.sink.endpoint.sslContextParameters": , - - // The hostname. The default value is 0.0.0.0 - // "camel.component.websocket.host": "0.0.0.0", - - // The port number. The default value is 9292 - // "camel.component.websocket.port": 9292, - - // Whether the producer should be started lazy (on the first message). By starting lazy you can - // use this to allow CamelContext and routes to startup in situations where a producer may otherwise - // fail during starting and cause the route to fail being started. By deferring this startup to be - // lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. - // Beware that when the first message is processed then creating and starting the producer may take a little - // time and prolong the total processing time of the processing. - // "camel.component.websocket.lazyStartProducer": false, - - // Whether autowiring is enabled. This is used for automatic autowiring options (the option must - // be marked as autowired) by looking up in the registry to find if there is a single instance - // of matching type, which then gets configured on the component. This can be used for automatic - // configuring JDBC data sources, JMS connection factories, AWS Clients, etc. - // "camel.component.websocket.autowiredEnabled": true, - - // If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX - // support for more details. - // "camel.component.websocket.enableJmx": false, - - // To set a value for maximum number of threads in server thread pool. MaxThreads\/minThreads - // or threadPool fields are required due to switch to Jetty9. The default values for maxThreads - // is 1 2 noCores. - // "camel.component.websocket.maxThreads": , - - // To set a value for minimum number of threads in server thread pool. MaxThreads\/minThreads - // or threadPool fields are required due to switch to Jetty9. The default values for minThreads - // is 1. - // "camel.component.websocket.minThreads": , - - // This is a comma-separated list of subprotocols that are supported by the application. The list - // is in priority order. The first subprotocol on this list that is proposed by the client is - // the one that will be accepted. If no subprotocol on this list is proposed by the client, then the - // websocket connection is refused. The special value 'any' means that any subprotocol is acceptable. 'any' - // can be used on its own, or as a failsafe at the end of a list of more specific protocols. 'any' will also - // match the case where no subprotocol is proposed by the client. - // "camel.component.websocket.subprotocol": "any", - - // To use a custom thread pool for the server. MaxThreads\/minThreads or threadPool fields are - // required due to switch to Jetty9. - // "camel.component.websocket.threadPool": , - - // To configure security using SSLContextParameters - // "camel.component.websocket.sslContextParameters": , - - // The password for the keystore when using SSL. - // "camel.component.websocket.sslKeyPassword": , - - // The path to the keystore. - // "camel.component.websocket.sslKeystore": , - - // The password when using SSL. - // "camel.component.websocket.sslPassword": , - - // Enable usage of global SSL context parameters. - // "camel.component.websocket.useGlobalSslContextParameters": false, -} diff --git a/modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5 b/modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5 deleted file mode 100644 index 19837c3..0000000 --- a/modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5 +++ /dev/null @@ -1,84 +0,0 @@ -//Exchange data with external Websocket servers using Async Http Client -{ - name: "websocket-async-lua-objects", - "connector.class": "org.apache.camel.kafkaconnector.ahcws.CamelAhcwsSinkConnector", - "tasks.max": 1, - //use the kafka converters that better suit your needs, these are just defaults: - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.storage.StringConverter", - //https://docs.confluent.io/platform/current/installation/configuration/connect/sink-connect-configs.html#sinkconnectorconfigs_topics - //List of topics to consume, separated by commas - // topics= - "topics.regex": "^Lua.*$", - - // mandatory properties (for a complete properties list see the connector documentation): - - // The URI to use such as http://hostname:port/path - "camel.sink.path.httpUri": "host.docker.internal:8095/ws/lua", - // If the option is true, then the Exchange.HTTP_URI header is ignored, and use the endpoint's - // URI for request. You may also set the throwExceptionOnFailure to be false to let the - // AhcProducer send all the fault response back. - // camel.sink.endpoint.bridgeEndpoint= - - // The initial in-memory buffer size used when transferring data between Camel and AHC Client. - // camel.sink.endpoint.bufferSize= - // To use a custom HeaderFilterStrategy to filter header to and from Camel message. - // camel.sink.endpoint.headerFilterStrategy= - // Option to disable throwing the AhcOperationFailedException in case of failed responses from - // the remote server. This allows you to get all responses regardless of the HTTP status code. - // camel.sink.endpoint.throwExceptionOnFailure= - // If enabled and an Exchange failed processing on the consumer side, and if the caused - - // Exception was send back serialized in the response as a application\/x-java-serialized-object - // content type (for example using Jetty or Servlet Camel components). On the producer side the - // exception will be deserialized and thrown as is, instead of the AhcOperationFailedException. - // The caused exception is required to be serialized. This is by default turned off. - // If you enable this then be aware that Java will deserialize the incoming data from the - // request to Java and that can be a potential security risk. - // camel.sink.endpoint.transferException= - - // Define if the Connection Close header has to be added to HTTP Request. This parameter is false by default - // camel.sink.endpoint.connectionClose= - // Configure a cookie handler to maintain a HTTP session - // camel.sink.endpoint.cookieHandler= - - // Whether the producer should be started lazy (on the first message). By starting lazy you can - // use this to allow CamelContext and routes to startup in situations where a producer - // may otherwise fail during starting and cause the route to fail being started. By deferring - // this startup to be lazy then the startup failure can be handled during routing messages via - // Camel's routing error handlers. Beware that when the first message is processed then creating - // and starting the producer may take a little time and prolong the total processing time of the - // processing. - // camel.sink.endpoint.lazyStartProducer= - - // To enable streaming to send data as multiple text fragments. - // camel.sink.endpoint.useStreaming= - // To use a custom AhcBinding which allows to control how to bind between AHC and Camel. - // camel.sink.endpoint.binding= - // To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance. - // camel.sink.endpoint.clientConfig= - // To configure the AsyncHttpClientConfig using the key\/values from the Map. - // camel.sink.endpoint.clientConfigOptions= - // To configure the AsyncHttpClientConfig Realm using the key\/values from the Map. - // camel.sink.endpoint.clientConfigRealmOptions= - // Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. This reference overrides any configured SSLContextParameters at the component level. See Using the JSSE Configuration Utility. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level. - // camel.sink.endpoint.sslContextParameters= - // Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail during starting and cause the route to fail being started. By deferring this startup to be lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. Beware that when the first message is processed then creating and starting the producer may take a little time and prolong the total processing time of the processing. - // camel.component.ahc-ws.lazyStartProducer= - // Whether to allow java serialization when a request uses context-type=application\/x-java-serialized-object This is by default turned off. If you enable this then be aware that Java will deserialize the incoming data from the request to Java and that can be a potential security risk. - // camel.component.ahc-ws.allowJavaSerializedObject= - // Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching type, which then gets configured on the component. This can be used for automatic configuring JDBC data sources, JMS connection factories, AWS Clients, etc. - // camel.component.ahc-ws.autowiredEnabled= - // To use a custom AhcBinding which allows to control how to bind between AHC and Camel. - // camel.component.ahc-ws.binding= - // To use a custom AsyncHttpClient - // camel.component.ahc-ws.client= - // To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance. - // camel.component.ahc-ws.clientConfig= - // To use a custom org.apache.camel.spi.HeaderFilterStrategy to filter header to and from Camel message. - // camel.component.ahc-ws.headerFilterStrategy= - // Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level. - // camel.component.ahc-ws.sslContextParameters= - // Enable usage of global SSL context parameters. - // camel.component.ahc-ws.useGlobalSslContextParameters= -} diff --git a/settings.gradle.kts b/settings.gradle.kts index 37b5fca..fee9190 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,7 +10,6 @@ include( ":modules:infra-kafka-cluster", ":modules:infra-kafka-pipe", - ":modules:infra-kafka-connect", ":modules:kt-rcon", ":modules:web-map", From dacb6f62c488f1c95f116c364f66de831b124ecd Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:56:53 +0200 Subject: [PATCH 03/22] tidy up Gradle config, standardise a group for Factorio Mod tasks --- .../FactorioModPortalPublishClient.kt | 4 +- .../{ => portal}/FactorioModPublishTask.kt | 5 +- .../factoriomod/{ => portal}/model.kt | 2 +- .../adamko/kafkatorio/factoriomod/tasks.kt | 12 +++ .../kafkatorio/lang/kotlin-jvm.gradle.kts | 8 -- ...criptTask.kt => GenerateTypeScriptTask.kt} | 2 +- modules/events-mod/build.gradle.kts | 7 +- modules/events-schema/build.gradle.kts | 79 ++---------------- modules/events-server/build.gradle.kts | 10 +-- .../infra-factorio-client/build.gradle.kts | 20 +++-- .../infra-factorio-server/build.gradle.kts | 16 +++- modules/web-map/build.gradle.kts | 83 +++++-------------- settings.gradle.kts | 9 -- 13 files changed, 83 insertions(+), 174 deletions(-) rename buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/{ => portal}/FactorioModPortalPublishClient.kt (97%) rename buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/{ => portal}/FactorioModPublishTask.kt (95%) rename buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/{ => portal}/model.kt (98%) create mode 100644 buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/tasks.kt rename buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/{GenerateTypescriptTask.kt => GenerateTypeScriptTask.kt} (91%) diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPortalPublishClient.kt b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPortalPublishClient.kt similarity index 97% rename from buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPortalPublishClient.kt rename to buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPortalPublishClient.kt index cb920fe..7b1aead 100644 --- a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPortalPublishClient.kt +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPortalPublishClient.kt @@ -1,4 +1,4 @@ -package dev.adamko.kafkatorio.factoriomod +package dev.adamko.kafkatorio.factoriomod.portal import io.ktor.client.HttpClient import io.ktor.client.engine.cio.CIO @@ -98,7 +98,7 @@ class FactorioModPortalPublishClient( require(response.status.isSuccess()) { "init upload request failed" } return when (initUploadResponse) { - is Failure -> error(initUploadResponse) + is Failure -> error(initUploadResponse) is InitUploadResponse.Success -> initUploadResponse } } diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPublishTask.kt b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPublishTask.kt similarity index 95% rename from buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPublishTask.kt rename to buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPublishTask.kt index 3a58751..d798e32 100644 --- a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPublishTask.kt +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPublishTask.kt @@ -1,5 +1,6 @@ -package dev.adamko.kafkatorio.factoriomod +package dev.adamko.kafkatorio.factoriomod.portal +import dev.adamko.kafkatorio.factoriomod.FactorioMod import io.ktor.http.URLBuilder import io.ktor.http.takeFrom import javax.inject.Inject @@ -59,7 +60,7 @@ abstract class FactorioModPublishTask @Inject constructor( init { - group = "factorio" + group = FactorioMod.TASK_GROUP } diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/model.kt b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/model.kt similarity index 98% rename from buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/model.kt rename to buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/model.kt index 8fad949..aee2ac6 100644 --- a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/model.kt +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/model.kt @@ -1,4 +1,4 @@ -package dev.adamko.kafkatorio.factoriomod +package dev.adamko.kafkatorio.factoriomod.portal import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/tasks.kt b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/tasks.kt new file mode 100644 index 0000000..b657ace --- /dev/null +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/tasks.kt @@ -0,0 +1,12 @@ +package dev.adamko.kafkatorio.factoriomod + +object FactorioMod { + + const val TASK_GROUP = "factorioMod" + + /** Lifecycle task for publishing the mod locally */ + const val PUBLISH_MOD_LOCAL_TASK_NAME = "publishModLocal" + + const val PUBLISH_MOD_TASK_NAME = "publishMod" + +} diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts index c0d902b..eb96e83 100644 --- a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts @@ -12,16 +12,8 @@ val projectJvmTarget = "11" dependencies { - -// implementation(platform("org.jetbrains.kotlin:kotlin-bom")) - -// testImplementation(platform("org.junit:junit-bom:${Versions.junit}")) testImplementation("org.junit.jupiter:junit-jupiter") -// testRuntimeOnly("org.junit.platform:junit-platform-launcher") { -// because("Only needed to run tests in a version of IntelliJ IDEA that bundles older versions") -// } -// testImplementation(platform("io.kotest:kotest-bom:${Versions.kotest}")) testImplementation("io.kotest:kotest-runner-junit5") testImplementation("io.kotest:kotest-assertions-core") testImplementation("io.kotest:kotest-property") diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypescriptTask.kt b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypeScriptTask.kt similarity index 91% rename from buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypescriptTask.kt rename to buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypeScriptTask.kt index 51b7032..fab29ae 100644 --- a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypescriptTask.kt +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypeScriptTask.kt @@ -8,7 +8,7 @@ import org.gradle.api.tasks.OutputDirectory import org.gradle.api.tasks.TaskAction -abstract class GenerateTypescriptTask @Inject constructor( +abstract class GenerateTypeScriptTask @Inject constructor( private val fileOps: FileSystemOperations, ) : JavaExec() { diff --git a/modules/events-mod/build.gradle.kts b/modules/events-mod/build.gradle.kts index 0a37eff..567fac2 100644 --- a/modules/events-mod/build.gradle.kts +++ b/modules/events-mod/build.gradle.kts @@ -1,4 +1,4 @@ -import dev.adamko.kafkatorio.factoriomod.FactorioModPublishTask +import dev.adamko.kafkatorio.factoriomod.portal.FactorioModPublishTask import dev.adamko.kafkatorio.gradle.asConsumer import dev.adamko.kafkatorio.gradle.asProvider import dev.adamko.kafkatorio.gradle.dropDirectory @@ -43,7 +43,7 @@ val projectTokensX = projectTokens.apply { put("mod.name", modName) put("mod.title", "Kafkatorio Events") put("mod.description", modDescription) - put("factorio.version", modFactorioCompatibility.get()) + put("factorio.version", modFactorioCompatibility) } val tsSrcDir: Directory = layout.projectDirectory.dir("src/main/typescript") @@ -139,7 +139,8 @@ distributions { from(licenseFile) from(typescriptToLua.map { it.outputDirectory }) filesNotMatching("**/*.png") { - filter("tokens" to projectTokens.get()) + val tokens: MutableMap = projectTokens.get() + filter("tokens" to tokens) } includeEmptyDirs = false exclude { diff --git a/modules/events-schema/build.gradle.kts b/modules/events-schema/build.gradle.kts index 2a1907f..31b22fa 100644 --- a/modules/events-schema/build.gradle.kts +++ b/modules/events-schema/build.gradle.kts @@ -1,6 +1,6 @@ import dev.adamko.kafkatorio.gradle.asProvider import dev.adamko.kafkatorio.gradle.typescriptAttributes -import dev.adamko.kafkatorio.task.GenerateTypescriptTask +import dev.adamko.kafkatorio.task.GenerateTypeScriptTask plugins { @@ -14,25 +14,8 @@ plugins { kotlin { js(IR) { -// binaries.executable() browser {} - // browser{} causes this error: - // Cannot load "webpack", it is not registered! - // Perhaps you are missing some plugin? - // Cannot load "sourcemap", it is not registered! - // Perhaps you are missing some plugin? - // Server start failed on port 9876: Error: No provider for "framework:mocha"! (Resolving: framework:mocha) - // browser { - // runTask { - // sourceMaps = true - // } - // testTask { - // useKarma { - // useChromeHeadless() - // } - // } - // } -// nodejs() + // this is a library - don't set binaries.executable() } jvm { val main by compilations.getting { @@ -75,9 +58,6 @@ kotlin { implementation(project.dependencies.platform(libs.kotest.bom)) implementation("io.kotest:kotest-framework-engine") -// implementation("io.kotest:kotest-assertions-core") -// implementation("io.kotest:kotest-property") -// implementation("io.kotest:kotest-assertions-json") implementation(libs.kotest.core) implementation(libs.kotest.datatest) implementation(libs.kotest.prop) @@ -88,10 +68,7 @@ kotlin { val jvmMain by getting { dependencies { implementation("com.github.aSemy:ts-generator:v1.2.1") -// implementation("com.github.ntrrgc:ts-generator:1.1.2") implementation(kotlin("reflect")) - -// implementation("com.github.adamko-dev:kotlinx-serialization-typescript-generator:0.0.5") } } @@ -114,7 +91,7 @@ val mainCompilation: Provider = kotlin.jvm().compilations.named("main").map { it.runtimeDependencyFiles } -val generateTypescript by tasks.registering(GenerateTypescriptTask::class) { +val generateTypeScript by tasks.registering(GenerateTypeScriptTask::class) { dependsOn(jvmJar) classpath( jvmJar, @@ -125,68 +102,26 @@ val generateTypescript by tasks.registering(GenerateTypescriptTask::class) { args(temporaryDir.canonicalPath) } -//val generateTypescript2 by tasks.registering(JavaExec::class) { -// group = "kt2ts" -// -// dependsOn(jvmJar) -// -// inputs.files(mainCompilation) -// -// classpath( -// jvmJar, -// mainCompilation, -// ) -// mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2ts2Kt") -// -// args(temporaryDir) -// -// val buildOutput = layout.buildDirectory.dir("generated/typescript") -// outputs.dir(buildOutput) -// -// doFirst { -// delete(temporaryDir) -// mkdir(temporaryDir) -// } -// -// doLast { -// sync { -// from(temporaryDir) -// into(buildOutput) -// include("**/*.ts") -// } -// } -//} - val schemaTsDistributionName: Provider = providers.provider { "${rootProject.name}-${project.name}" } -val generateTypescriptOutputFiles: Provider = - generateTypescript.map { it.outputs.files.asFileTree } +val generateTypeScriptOutputFiles: Provider = + generateTypeScript.map { it.outputs.files.asFileTree } val schemaTs by distributions.registering { distributionBaseName.set(schemaTsDistributionName) contents { - from(generateTypescriptOutputFiles) + from(generateTypeScriptOutputFiles) } } val schemaTsZipTask: TaskProvider = tasks.named("${schemaTs.name}DistZip") val schemaTsZipTaskArchiveFile: Provider = schemaTsZipTask.flatMap { it.archiveFile } -val typescriptModelGenerated: Configuration by configurations.creating { +val typeScriptModelGenerated: Configuration by configurations.creating { asProvider() typescriptAttributes(objects) outgoing.artifact(schemaTsZipTaskArchiveFile) } - - -//tasks.matching { -// it.name == "jsGenerateExternalsIntegrated" -//}.configureEach { -// @Suppress("UnstableApiUsage") -// notCompatibleWithConfigurationCache("try to prevent 'Projects must be configuring' error") -//// outputs.upToDateWhen { true } -// dependsOn(":kotlinNpmInstall") -//} diff --git a/modules/events-server/build.gradle.kts b/modules/events-server/build.gradle.kts index 25580f9..7978403 100644 --- a/modules/events-server/build.gradle.kts +++ b/modules/events-server/build.gradle.kts @@ -1,12 +1,14 @@ import dev.adamko.kafkatorio.task.KafkaConsumerGroupsTask import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + plugins { dev.adamko.kafkatorio.lang.`kotlin-jvm` kotlin("plugin.serialization") application } + description = """ Receive raw Factorio Events from Kafka and process them into targeted topics or data formats. @@ -17,6 +19,7 @@ val projectId: String by project.extra val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state") + dependencies { implementation(platform(projects.modules.versionsPlatform)) @@ -49,9 +52,6 @@ dependencies { implementation(libs.bundles.hoplite) } -//configurations.all { -// resolutionStrategy.cacheDynamicVersionsFor(1, TimeUnit.MINUTES) -//} application { mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt") @@ -93,7 +93,3 @@ idea { excludeDirs = excludeDirs + kafkaStateDir.asFile } } - -//tasks.withType().all { -// notCompatibleWithConfigurationCache("error: Projects must be configuring") -//} diff --git a/modules/infra-factorio-client/build.gradle.kts b/modules/infra-factorio-client/build.gradle.kts index 779c712..33d76a1 100644 --- a/modules/infra-factorio-client/build.gradle.kts +++ b/modules/infra-factorio-client/build.gradle.kts @@ -1,3 +1,4 @@ +import dev.adamko.kafkatorio.factoriomod.FactorioMod import dev.adamko.kafkatorio.gradle.asConsumer import dev.adamko.kafkatorio.gradle.factorioModAttributes import dev.adamko.kafkatorio.gradle.not @@ -34,10 +35,11 @@ dependencies { factorioMod(projects.modules.eventsMod) } + // val deployModToClient by tasks.registering(Copy::class) { description = "Copy the mod to the Factorio client" - group = project.name + group = FactorioMod.TASK_GROUP onlyIf { clientModsDirectory.orNull?.asFile?.exists() == true } @@ -50,13 +52,14 @@ val deployModToClient by tasks.registering(Copy::class) { } // + // fun ExecOperations.isFactorioRunning(): Spec = ProcessRunningSpec(this, "factorio.exe") val clientLaunch by tasks.registering(Exec::class) { - description = "Run Factorio Steam game client" - group = project.name + description = "Run local Factorio Steam game client" + group = FactorioMod.TASK_GROUP onlyIf(!serviceOf().isFactorioRunning()) onlyIf { steamExe.orNull?.asFile?.exists() == true } @@ -74,8 +77,8 @@ val clientLaunch by tasks.registering(Exec::class) { } val clientKill by tasks.registering(Exec::class) { - description = "Run Factorio Steam game client" - group = project.name + description = "Stop the local Factorio Steam game client" + group = FactorioMod.TASK_GROUP onlyIf(serviceOf().isFactorioRunning()) @@ -84,7 +87,12 @@ val clientKill by tasks.registering(Exec::class) { } // -tasks.build { dependsOn(deployModToClient) } + +tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) { + group = FactorioMod.TASK_GROUP + dependsOn(deployModToClient) +} + tasks.processRun { dependsOn( diff --git a/modules/infra-factorio-server/build.gradle.kts b/modules/infra-factorio-server/build.gradle.kts index 7d75285..158e151 100644 --- a/modules/infra-factorio-server/build.gradle.kts +++ b/modules/infra-factorio-server/build.gradle.kts @@ -1,3 +1,4 @@ +import dev.adamko.kafkatorio.factoriomod.FactorioMod import dev.adamko.kafkatorio.gradle.asConsumer import dev.adamko.kafkatorio.gradle.factorioModAttributes import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs @@ -6,21 +7,25 @@ plugins { dev.adamko.kafkatorio.infra.`docker-compose` } + val dockerSrcDir: Directory by extra val factorioServerDataDir: Directory = dockerSrcDir.dir("factorio-server") + val factorioMod: Configuration by configurations.creating { asConsumer() factorioModAttributes(objects) } + dependencies { factorioMod(projects.modules.eventsMod) } + val deployModToServer by tasks.registering(Copy::class) { description = "Copy the mod to the Factorio Docker server" - group = project.name + group = FactorioMod.TASK_GROUP dependsOn(factorioMod) @@ -34,10 +39,12 @@ val deployModToServer by tasks.registering(Copy::class) { } } + tasks.dockerDown { commandLine = parseSpaceSeparatedArgs(""" docker-compose stop """) } + tasks.dockerUp { dependsOn( deployModToServer, @@ -45,11 +52,16 @@ tasks.dockerUp { ) } + tasks.dockerEnv { properties("FACTORIO_VERSION" to libs.versions.factorio.get()) } -tasks.build { dependsOn(deployModToServer) } + +tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) { + group = FactorioMod.TASK_GROUP + dependsOn(deployModToServer) +} idea { diff --git a/modules/web-map/build.gradle.kts b/modules/web-map/build.gradle.kts index a714375..8a02c87 100644 --- a/modules/web-map/build.gradle.kts +++ b/modules/web-map/build.gradle.kts @@ -1,4 +1,3 @@ -import dev.adamko.kafkatorio.gradle.taskProvider import org.jetbrains.kotlin.gradle.plugin.KotlinDependencyHandler import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackConfig @@ -17,6 +16,7 @@ kvision { enableHiddenKotlinJsStore.set(false) } + rootProject.extensions.configure { // https://github.com/rjaros/kvision/issues/410 versions.webpackCli.version = "4.10.0" @@ -48,13 +48,9 @@ afterEvaluate { kotlin { js(IR) { -// browser() -// binaries.executable() -// } -// -// js("frontend", IR) { -// nodejs() browser { + binaries.executable() + runTask { outputFileName = "main.bundle.js" sourceMaps = false @@ -87,16 +83,11 @@ kotlin { } } } - binaries.executable() // compilations["main"].packageJson { // this.devDependencies["webpack-dev-server"] = "4.9.1" // } } -// jvm("backend") { -// -// } - sourceSets { all { @@ -106,7 +97,6 @@ kotlin { } } -// val commonMain by getting { main { dependencies { implementation(dependencies.platform(libs.kotlinx.coroutines.bom)) @@ -117,13 +107,6 @@ kotlin { implementation(projects.modules.eventsSchema) implementation(libs.kotlinx.html) -// } -// } - -// val jsMain by getting { -// dependsOn(commonMain) -// -// dependencies { implementation(projects.modules.eventsSchema) @@ -154,37 +137,15 @@ kotlin { // implementation(dependencies.platform(devNpm("webpack-dev-server", "4.9.1"))) } -// val webDir = file("src/main/web") resources.srcDir("src/main/web") -// resources.srcDir("src/main/resources") - -// kotlin.srcDir("src/main/kotlin") } -// val frontendMain by getting { -// dependsOn(jsMain) -//// resources.srcDir("src/main/web") -//// resources.srcDir("src/main/resources") -// } -// -// val jsTest by getting { -// dependencies { -// implementation(kotlin("test-js")) -// -// kvision("kvision-testutils") -// -//// implementation(npm("karma", "^6.3.16")) -// } -// -// kotlin.srcDir("src/test/kotlin") -// } test { dependencies { implementation(kotlin("test-js")) kvision("kvision-testutils") - // implementation(npm("karma", "^6.3.16")) } } @@ -221,22 +182,22 @@ fun KotlinDependencyHandler.kvision( //evaluationDependsOn(projects.modules.eventsSchema.dependencyProject.path) -// stop warnings when building -tasks.configureEach { - when (name) { - "productionExecutableCompileSync" -> - dependsOn(taskProvider("compileProductionExecutableKotlinJs")) - "testTestDevelopmentExecutableCompileSync" -> - dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinJs")) -// "jsTestTestDevelopmentExecutableCompileSync" -> -// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinFrontend")) -// "jsProductionExecutableCompileSync" -> -// dependsOn(taskProvider("compileProductionExecutableKotlinFrontend")) - "browserDevelopmentRun" -> - dependsOn(taskProvider("browserDevelopmentWebpack")) - "browserDevelopmentWebpack" -> - dependsOn(taskProvider("assemble")) - "browserProductionWebpack" -> - dependsOn(taskProvider("developmentExecutableCompileSync")) - } -} +//// stop warnings when building +//tasks.configureEach { +// when (name) { +// "productionExecutableCompileSync" -> +// dependsOn(taskProvider("compileProductionExecutableKotlinJs")) +// "testTestDevelopmentExecutableCompileSync" -> +// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinJs")) +//// "jsTestTestDevelopmentExecutableCompileSync" -> +//// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinFrontend")) +//// "jsProductionExecutableCompileSync" -> +//// dependsOn(taskProvider("compileProductionExecutableKotlinFrontend")) +// "browserDevelopmentRun" -> +// dependsOn(taskProvider("browserDevelopmentWebpack")) +// "browserDevelopmentWebpack" -> +// dependsOn(taskProvider("assemble")) +// "browserProductionWebpack" -> +// dependsOn(taskProvider("developmentExecutableCompileSync")) +// } +//} diff --git a/settings.gradle.kts b/settings.gradle.kts index fee9190..b39fab1 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -25,14 +25,5 @@ apply(from = "./buildSrc/repositories.settings.gradle.kts") @Suppress("UnstableApiUsage") // Central declaration of repositories is an incubating feature dependencyResolutionManagement { - repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS) - -// repositories { -// maven("https://s01.oss.sonatype.org/content/repositories/snapshots/") { -//// content { -//// includeGroup("dev.adamko.kxstsgen") -//// } -// } -// } } From c9df87e5e0545bcc6ba09f09f1dcc0291eda7a56 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:57:14 +0200 Subject: [PATCH 04/22] fix NPE for config update event --- .../typescript/main/config-update/handleConfigurationUpdate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts b/modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts index 109d5b7..4808d6c 100644 --- a/modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts +++ b/modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts @@ -38,7 +38,7 @@ function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] { return modNames.map((modName) => { - let currentVer = script.active_mods[modName] ?? e.mod_changes[modName].new_version ?? null + let currentVer = script.active_mods[modName] ?? e.mod_changes[modName]?.new_version ?? null let previousVer = e.mod_changes[modName]?.old_version ?? null return { From 82ad2d27639114a1e5fd6e457e4205620966d9b1 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:57:43 +0200 Subject: [PATCH 05/22] remove more unused Kafka Connect stuff --- .../src/docker-compose.yml | 14 -------------- .../infra-kafka-cluster/src/kafka.Dockerfile | 19 ------------------- 2 files changed, 33 deletions(-) diff --git a/modules/infra-kafka-cluster/src/docker-compose.yml b/modules/infra-kafka-cluster/src/docker-compose.yml index 6757307..8840793 100644 --- a/modules/infra-kafka-cluster/src/docker-compose.yml +++ b/modules/infra-kafka-cluster/src/docker-compose.yml @@ -46,20 +46,6 @@ services: KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: "http://connect:8083" # - KAFKA_CLUSTERS_0_ZOOKEEPER=kafka:9093 -# kafka-connect: -# image: dev-adamko/kafka-connect -# container_name: "kafka-connect" -# build: -# dockerfile: "kafka.Dockerfile" -# target: "kafka-connect" -# args: -# KAFKA_DL_URL: "https://dlcdn.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz" -# depends_on: -# - kafka-kraft -# ports: -# - "8083:8083" -# - # grafana: # image: "grafana/grafana:${GRAFANA_VERSION}" # container_name: "grafana" diff --git a/modules/infra-kafka-cluster/src/kafka.Dockerfile b/modules/infra-kafka-cluster/src/kafka.Dockerfile index 9d09f05..7670ec8 100644 --- a/modules/infra-kafka-cluster/src/kafka.Dockerfile +++ b/modules/infra-kafka-cluster/src/kafka.Dockerfile @@ -40,22 +40,3 @@ COPY --chmod=755 ./entrypoint.sh ./entrypoint.sh # launch the broker in KRaft mode, which means that it runs without ZooKeeper ENTRYPOINT ["/kafka/entrypoint.sh"] - - -### Kafka Connect ## -#FROM openjdk:11 as kafka-connect -## https://github.com/confluentinc/cp-docker-images/blob/5.3.3-post/debian/kafka-connect-base/Dockerfile -# -#ENV COMPONENT=kafka-connect -# -#WORKDIR /kafka-connect -# -#COPY --from=Kafka-Download /kafka . -# -## Default kafka-connect rest.port -#EXPOSE 8083 -# -#VOLUME ["/etc/${COMPONENT}/jars", "/etc/${COMPONENT}/secrets"] -# -## launch the broker in KRaft mode, which means that it runs without ZooKeeper -#ENTRYPOINT ["./bin/connect-standalone.sh", "./config/connect-standalone.properties"] From 4d0ec1a1a71612248fa8eb87420cd945b8f690b9 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:58:19 +0200 Subject: [PATCH 06/22] version bump 0.6.1 --- build.gradle.kts | 4 ++-- modules/events-mod/package-lock.json | 2 +- modules/events-mod/package.json | 2 +- modules/infra-factorio-server/src/.env | 2 +- modules/infra-kafka-cluster/src/.env | 2 +- modules/infra-kafka-pipe/src/.env | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 80d7379..fd9e82d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,11 +8,11 @@ plugins { `build-dashboard` // kotlin("multiplatform") apply false // try to stop error 'Projects must be configuring' -// kotlin("js") apply false // try to stop error 'Projects must be configuring' + kotlin("js") apply false // try to stop error 'Projects must be configuring' } group = "dev.adamko.kafkatorio" -version = "0.6.0" +version = "0.6.1" val licenseFile: RegularFile by extra(layout.projectDirectory.file("LICENSE")) diff --git a/modules/events-mod/package-lock.json b/modules/events-mod/package-lock.json index 7e847ad..785e0ba 100644 --- a/modules/events-mod/package-lock.json +++ b/modules/events-mod/package-lock.json @@ -1,6 +1,6 @@ { "name": "kafkatorio-events-mod", - "version": "0.6.0", + "version": "0.6.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/modules/events-mod/package.json b/modules/events-mod/package.json index be60289..4b3a926 100644 --- a/modules/events-mod/package.json +++ b/modules/events-mod/package.json @@ -1,6 +1,6 @@ { "name": "kafkatorio-events-mod", - "version": "0.6.0", + "version": "0.6.1", "private": true, "scripts": { "build": "tstl", diff --git a/modules/infra-factorio-server/src/.env b/modules/infra-factorio-server/src/.env index c532798..da16a1f 100644 --- a/modules/infra-factorio-server/src/.env +++ b/modules/infra-factorio-server/src/.env @@ -1,4 +1,4 @@ # Do not edit manually. This file was created with task 'dockerEnv' COMPOSE_PROJECT_NAME=kafkatorio FACTORIO_VERSION=1.1.59 -KAFKATORIO_VERSION=0.6.0 +KAFKATORIO_VERSION=0.6.1 diff --git a/modules/infra-kafka-cluster/src/.env b/modules/infra-kafka-cluster/src/.env index 04f99aa..d33aea8 100644 --- a/modules/infra-kafka-cluster/src/.env +++ b/modules/infra-kafka-cluster/src/.env @@ -1,7 +1,7 @@ # Do not edit manually. This file was created with task 'dockerEnv' COMPOSE_PROJECT_NAME=kafkatorio GRAFANA_VERSION=8.5.0 -KAFKATORIO_VERSION=0.6.0 +KAFKATORIO_VERSION=0.6.1 KAFKA_DL_URL=https\://dlcdn.apache.org/kafka/3.2.0/kafka_2.13-3.2.0.tgz KAFKA_VERSION=3.2.0 PROMETHEUS_VERSION=v2.34.0 diff --git a/modules/infra-kafka-pipe/src/.env b/modules/infra-kafka-pipe/src/.env index c97ae67..4a03adb 100644 --- a/modules/infra-kafka-pipe/src/.env +++ b/modules/infra-kafka-pipe/src/.env @@ -1,3 +1,3 @@ # Do not edit manually. This file was created with task 'dockerEnv' COMPOSE_PROJECT_NAME=kafkatorio -KAFKATORIO_VERSION=0.6.0 +KAFKATORIO_VERSION=0.6.1 From f4744c728645fc0e1464ee215289c73d9cb20ca4 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:58:38 +0200 Subject: [PATCH 07/22] webmap - fix max zoom --- .../web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt b/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt index a124582..8c40db5 100644 --- a/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt +++ b/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt @@ -125,7 +125,7 @@ class FactorioMap( minZoom = -2 maxZoom = 6 - maxNativeZoom = -1 + maxNativeZoom = 0 minNativeZoom = -1 noWrap = true From 275fb27a292947ef078db5e58b30bf52188f05e2 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:59:09 +0200 Subject: [PATCH 08/22] Kafka Stream instance -> stop coroutine only when NOT_RUNNING --- .../dev/adamko/kafkatorio/processor/KafkatorioTopology.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt index 8da6383..1aa3e53 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt @@ -105,8 +105,11 @@ internal class KafkatorioTopology( streams.setStateListener { newState: KafkaStreams.State, _ -> when (newState) { KafkaStreams.State.NOT_RUNNING, - KafkaStreams.State.ERROR -> cont.resume(newState) - else -> Unit // do nothing +// KafkaStreams.State.ERROR, + -> if (cont.isActive) { + cont.resume(newState) + } + else -> Unit // do nothing } } From b8c1c57a17a890fc0ab9e25ca81048c7f65bd61f Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:30:03 +0200 Subject: [PATCH 09/22] try emitting player disconnects and immediate packet updates immediately (otherwise the server doesn't see the disconnect message and the last player remains online) --- .../typescript/main/cache/EventDataCache.ts | 27 +++++++++++++- .../typescript/main/cache/cacheChecker.ts | 14 +++----- .../main/events/onPlayerEventListeners.ts | 35 ++++++++++++++----- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts b/modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts index e80a90a..17bcabb 100644 --- a/modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts +++ b/modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts @@ -1,5 +1,6 @@ import KafkatorioSettings from "../settings/KafkatorioSettings"; import {KafkatorioKeyedPacketData} from "../types"; +import packetEmitter from "../PacketEmitter"; export namespace EventUpdates { @@ -71,6 +72,10 @@ export namespace EventUpdates { if (expirationDurationTicks != undefined) { entry.expirationDurationTicks = expirationDurationTicks } + + if (entry.expirationDurationTicks != undefined && entry.expirationDurationTicks <= 0) { + this.extractAndEmitExpiredPackets() + } } @@ -90,8 +95,9 @@ export namespace EventUpdates { } + /** Removes all expired entries from the cache, and return them. */ // use in/out v4.7.0 https://github.com/microsoft/TypeScript/pull/48240 - public extractExpired(): Array { + private extractExpired(): Array { let countExpired = 0 let countTotal = 0 const expiredData: Array = [] @@ -114,6 +120,25 @@ export namespace EventUpdates { } + /** + * Removes and emits all expired packets. + * + * @return the number of emitted packets + */ + public extractAndEmitExpiredPackets() { + const cachedEvents: Array = this.extractExpired() + + if (cachedEvents.length > 0) { + log(`[EventDataCache] emitting ${cachedEvents.length} events on tick ${game.tick}`) + for (const event of cachedEvents) { + packetEmitter.emitKeyedPacket(event) + } + } + + return cachedEvents.length + } + + private getCacheEntry( key: EventUpdates.PacketKey, type: EventUpdates.PacketType, diff --git a/modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts b/modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts index febd47e..face16a 100644 --- a/modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts +++ b/modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts @@ -1,16 +1,10 @@ import EventCacheService from "./EventDataCache"; -import packetEmitter from "../PacketEmitter"; -import {KafkatorioKeyedPacketData} from "../types"; + /** Every 0.1 seconds */ script.on_nth_tick(6, (nthTick: NthTickEventData) => { - const cachedEvents: Array = EventCacheService.extractExpired() - - if (cachedEvents.length > 0) { - log(`nth tick ${nthTick.tick} has ${cachedEvents.length} events`) - - for (const event of cachedEvents) { - packetEmitter.emitKeyedPacket(event) - } + const emittedCount = EventCacheService.extractAndEmitExpiredPackets() + if (emittedCount > 0) { + log(`[on_nth_tick:${nthTick.tick}] emitted ${emittedCount} events`) } }) diff --git a/modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts b/modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts index b6583a3..22ad180 100644 --- a/modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts +++ b/modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts @@ -1,7 +1,11 @@ import {Converters} from "./converters"; import EventUpdatesManager from "../cache/EventDataCache"; -import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +import { + KafkatorioPacketData, + PlayerUpdateKey +} from "../../generated/kafkatorio-schema/kafkatorio-schema"; import Type = KafkatorioPacketData.Type; +import packetEmitter from "../PacketEmitter"; type PlayerUpdater = (player: LuaPlayer, data: KafkatorioPacketData.PlayerUpdate) => void @@ -44,7 +48,7 @@ function playerUpdateThrottle( data.events[eventName] ??= [] data.events[eventName].push(event.tick) }, - expirationDurationTicks + expirationDurationTicks, ) } @@ -183,13 +187,26 @@ script.on_event( defines.events.on_pre_player_left_game, (event: OnPrePlayerLeftGameEvent) => { log(`on_pre_player_left_game ${event.tick} ${event.name}`) - playerUpdateImmediate( - event, - (player, data) => { - data.disconnectReason = disconnectReasons.get(event.reason) - playerOnlineInfo(player, data) - } - ) + + const playerUpdateKey: PlayerUpdateKey = { + index: event.player_index + } + + const playerUpdate: KafkatorioPacketData.PlayerUpdate = { + type: KafkatorioPacketData.Type.PlayerUpdate, + key: playerUpdateKey, + disconnectReason: disconnectReasons.get(event.reason) + } + + packetEmitter.emitKeyedPacket(playerUpdate) + + // playerUpdateImmediate( + // event, + // (player, data) => { + // data.disconnectReason = disconnectReasons.get(event.reason) + // playerOnlineInfo(player, data) + // } + // ) } ) From f3f92afd22ee829f037ced97ca36850e1ccafeeb Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:30:30 +0200 Subject: [PATCH 10/22] try fixing weird gradle getProjectTokens() null null null bug --- modules/events-mod/build.gradle.kts | 2 +- modules/events-mod/package.json | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/events-mod/build.gradle.kts b/modules/events-mod/build.gradle.kts index 567fac2..c66cc78 100644 --- a/modules/events-mod/build.gradle.kts +++ b/modules/events-mod/build.gradle.kts @@ -139,7 +139,7 @@ distributions { from(licenseFile) from(typescriptToLua.map { it.outputDirectory }) filesNotMatching("**/*.png") { - val tokens: MutableMap = projectTokens.get() + val tokens: MutableMap = projectTokensX.get() filter("tokens" to tokens) } includeEmptyDirs = false diff --git a/modules/events-mod/package.json b/modules/events-mod/package.json index 4b3a926..50840cf 100644 --- a/modules/events-mod/package.json +++ b/modules/events-mod/package.json @@ -9,7 +9,8 @@ "engines": { "node": "~14.19.1" }, - "devDependencies": {}, + "devDependencies": { + }, "dependencies": { "lua-types": "^2.11.0", "typescript-to-lua": "1.6.0", @@ -17,7 +18,10 @@ "typescript": "4.7.4" }, "license": "UNLICENSED", - "peerDependencies": {}, - "optionalDependencies": {}, - "bundledDependencies": [] + "peerDependencies": { + }, + "optionalDependencies": { + }, + "bundledDependencies": [ + ] } From acbc15b6d4f3aa7d5868e8831ab67c677fbd0ad6 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:31:54 +0200 Subject: [PATCH 11/22] map chunk grouping: reduce the amount of tiles generated (thanks to pixelated css), and reduce debounce duration --- .../processor/misc/DebounceProcessor.kt | 14 +++++++-- .../processor/topology/colourMapChunks.kt | 31 +++++++++++-------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessor.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessor.kt index 2658751..9502e63 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessor.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessor.kt @@ -30,14 +30,17 @@ class DebounceProcessor( private val storeName: String, ) : Processor { + private val checkInterval: DurationJvm = checkInterval.toJavaDuration() private lateinit var state: State + override fun init(context: ProcessorContext) { state = Active(context) } + override fun process(record: Record) { (state as Active).apply { val (key, value, _) = record @@ -49,21 +52,28 @@ class DebounceProcessor( } } + override fun close() { (state as? Active)?.schedule?.cancel() state = Closed() } + private sealed interface State + private class Closed : State + private inner class Active( val context: ProcessorContext ) : State { val store: TimestampedKeyValueStore = context.getStateStore(storeName) - val schedule: Cancellable = - context.schedule(checkInterval, PunctuationType.WALL_CLOCK_TIME, ::checkRecords) + val schedule: Cancellable = context.schedule( + checkInterval, + PunctuationType.WALL_CLOCK_TIME, + ::checkRecords + ) private fun checkRecords(now: Long) { store.useAll { diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt index 2a171f0..c471d18 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt @@ -17,6 +17,7 @@ import dev.adamko.kotka.extensions.groupedAs import dev.adamko.kotka.extensions.materializedAs import dev.adamko.kotka.extensions.producedAs import dev.adamko.kotka.extensions.repartitionedAs +import dev.adamko.kotka.extensions.stream import dev.adamko.kotka.extensions.streams.filter import dev.adamko.kotka.extensions.streams.map import dev.adamko.kotka.extensions.streams.mapValues @@ -55,26 +56,30 @@ fun colourMapChunks(builder: StreamsBuilder): Topology { chunkTilesColouredTable032.streamMapChunkColouredTo( ChunkSize.CHUNK_032, - TOPIC_MAP_CHUNK_COLOURED_032_STATE + TOPIC_MAP_CHUNK_COLOURED_032_STATE, ) - val chunkTilesColoured032Debounced: KStream> = + val chunkTilesColoured: KStream> = builder.stream( - TOPIC_MAP_CHUNK_COLOURED_STATE, consumedAs( - "$pid.consume.map-chunks-coloured", - kxsBinary.serde(), - kxsBinary.serde(), + "$pid.consume.map-chunks-debounced", + kxsBinary.serde(ServerMapChunkId.serializer()), + kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())), ), + TOPIC_MAP_CHUNK_COLOURED_STATE, ) mapOf( - ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064, - ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128, - ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256, +// ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064, +// ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128, +// ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256, +// ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512, + + ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256, ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512, + ).forEach { (from, to) -> - chunkTilesColoured032Debounced + chunkTilesColoured .filter("$pid.combine-chunks.filter-$from-to-$to") { key, _ -> key.chunkSize == from }.groupByChunkPosition(to) @@ -86,9 +91,9 @@ fun colourMapChunks(builder: StreamsBuilder): Topology { namePrefix = "groupTilesMapChunks", sourceTopic = TOPIC_MAP_CHUNK_COLOURED_032_STATE, sinkTopic = TOPIC_MAP_CHUNK_COLOURED_STATE, - inactivityDuration = 15.seconds, - keySerde = kxsBinary.serde(), - valueSerde = kxsBinary.serde>(), + inactivityDuration = 5.seconds, + keySerde = kxsBinary.serde(ServerMapChunkId.serializer()), + valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())), ) } From 23dbbfee2799c925a8e13f7cbe2a45ebfb70e6a7 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:32:08 +0200 Subject: [PATCH 12/22] enable local port for Kafka UI --- modules/infra-kafka-cluster/src/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/infra-kafka-cluster/src/docker-compose.yml b/modules/infra-kafka-cluster/src/docker-compose.yml index 8840793..d60961f 100644 --- a/modules/infra-kafka-cluster/src/docker-compose.yml +++ b/modules/infra-kafka-cluster/src/docker-compose.yml @@ -36,8 +36,8 @@ services: - "traefik-nw" labels: - "traefik.enable=true" - # ports: - # - "8080:8080" + ports: + - "18090:8080" environment: KAFKA_CLUSTERS_0_NAME: "${COMPOSE_PROJECT_NAME}" KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka:29092" From 6ac89b3db09b9119b77f6376e01af915f0960659 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:33:03 +0200 Subject: [PATCH 13/22] events server: change default docker port to 8080 (maybe better Traefik config?) --- modules/events-server/docker/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/events-server/docker/docker-compose.yml b/modules/events-server/docker/docker-compose.yml index 02f1f6b..ee23c47 100644 --- a/modules/events-server/docker/docker-compose.yml +++ b/modules/events-server/docker/docker-compose.yml @@ -14,16 +14,16 @@ services: environment: KAFKA_BOOTSTRAP_SERVERS: "http://kafka:29092" # KAFKA_STREAMS_STATE_DIR: "http://kafka:29092" - SERVER_PORT: "12081" + SERVER_PORT: "8080" ports: - - "12080:12081" # http and ws + - "12080:8080" # http and ws networks: - "kafka-nw" - "kafkatorio-nw" - "traefik-nw" labels: - "traefik.enable=true" - - "traefik.http.services.events-server-kafkatorio.loadbalancer.server.port=12081" +# - "traefik.http.services.events-server-kafkatorio.loadbalancer.server.port=12081" networks: kafka-nw: From e7abf4583a747d9233b62fcee5549818f897c8aa Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:45:59 +0200 Subject: [PATCH 14/22] rename Factorio mod tasks --- modules/events-mod/build.gradle.kts | 2 +- modules/infra-factorio-client/build.gradle.kts | 8 ++++---- modules/infra-factorio-server/build.gradle.kts | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/events-mod/build.gradle.kts b/modules/events-mod/build.gradle.kts index c66cc78..7feaa3c 100644 --- a/modules/events-mod/build.gradle.kts +++ b/modules/events-mod/build.gradle.kts @@ -116,7 +116,7 @@ tasks.distZip { } -val publishMod by tasks.registering(FactorioModPublishTask::class) { +val publishModToPortal by tasks.registering(FactorioModPublishTask::class) { dependsOn(tasks.check) distributionZip.set(tasks.distZip.flatMap { it.archiveFile }) diff --git a/modules/infra-factorio-client/build.gradle.kts b/modules/infra-factorio-client/build.gradle.kts index 33d76a1..4c29a79 100644 --- a/modules/infra-factorio-client/build.gradle.kts +++ b/modules/infra-factorio-client/build.gradle.kts @@ -37,7 +37,7 @@ dependencies { // -val deployModToClient by tasks.registering(Copy::class) { +val deployModToLocalClient by tasks.registering(Copy::class) { description = "Copy the mod to the Factorio client" group = FactorioMod.TASK_GROUP @@ -65,7 +65,7 @@ val clientLaunch by tasks.registering(Exec::class) { onlyIf { steamExe.orNull?.asFile?.exists() == true } onlyIf { clientModsDirectory.orNull?.asFile?.exists() == true } - dependsOn(deployModToClient) + dependsOn(deployModToLocalClient) mustRunAfter(clientKill, ":modules:infra-factorio-server:processRun") commandLine = parseSpaceSeparatedArgs( @@ -90,13 +90,13 @@ val clientKill by tasks.registering(Exec::class) { tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) { group = FactorioMod.TASK_GROUP - dependsOn(deployModToClient) + dependsOn(deployModToLocalClient) } tasks.processRun { dependsOn( - deployModToClient, + deployModToLocalClient, clientLaunch, ) } diff --git a/modules/infra-factorio-server/build.gradle.kts b/modules/infra-factorio-server/build.gradle.kts index 158e151..11c61b7 100644 --- a/modules/infra-factorio-server/build.gradle.kts +++ b/modules/infra-factorio-server/build.gradle.kts @@ -23,7 +23,7 @@ dependencies { } -val deployModToServer by tasks.registering(Copy::class) { +val deployModToLocalServer by tasks.registering(Copy::class) { description = "Copy the mod to the Factorio Docker server" group = FactorioMod.TASK_GROUP @@ -47,7 +47,7 @@ tasks.dockerDown { tasks.dockerUp { dependsOn( - deployModToServer, + deployModToLocalServer, ":modules:infra-kafka-cluster:processRun", ) } @@ -60,7 +60,7 @@ tasks.dockerEnv { tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) { group = FactorioMod.TASK_GROUP - dependsOn(deployModToServer) + dependsOn(deployModToLocalServer) } From e078e6316dd98753d4941c4fd02cc21a828ecca3 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sun, 26 Jun 2022 23:50:30 +0200 Subject: [PATCH 15/22] manually bump Kafkatorio versions for events-server and web-map (I still need to automate that...) --- modules/events-mod/package.json | 12 ++++-------- modules/events-server/docker/.env | 2 +- modules/web-map/docker/.env | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/modules/events-mod/package.json b/modules/events-mod/package.json index 50840cf..4b3a926 100644 --- a/modules/events-mod/package.json +++ b/modules/events-mod/package.json @@ -9,8 +9,7 @@ "engines": { "node": "~14.19.1" }, - "devDependencies": { - }, + "devDependencies": {}, "dependencies": { "lua-types": "^2.11.0", "typescript-to-lua": "1.6.0", @@ -18,10 +17,7 @@ "typescript": "4.7.4" }, "license": "UNLICENSED", - "peerDependencies": { - }, - "optionalDependencies": { - }, - "bundledDependencies": [ - ] + "peerDependencies": {}, + "optionalDependencies": {}, + "bundledDependencies": [] } diff --git a/modules/events-server/docker/.env b/modules/events-server/docker/.env index c97ae67..4a03adb 100644 --- a/modules/events-server/docker/.env +++ b/modules/events-server/docker/.env @@ -1,3 +1,3 @@ # Do not edit manually. This file was created with task 'dockerEnv' COMPOSE_PROJECT_NAME=kafkatorio -KAFKATORIO_VERSION=0.6.0 +KAFKATORIO_VERSION=0.6.1 diff --git a/modules/web-map/docker/.env b/modules/web-map/docker/.env index c97ae67..4a03adb 100644 --- a/modules/web-map/docker/.env +++ b/modules/web-map/docker/.env @@ -1,3 +1,3 @@ # Do not edit manually. This file was created with task 'dockerEnv' COMPOSE_PROJECT_NAME=kafkatorio -KAFKATORIO_VERSION=0.6.0 +KAFKATORIO_VERSION=0.6.1 From e9a4ee4c48dfa41465262ca0eb75dbb996531307 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:07:42 +0200 Subject: [PATCH 16/22] set up syslog docker parser (remove need for Kafka Pipe script) --- build.gradle.kts | 2 +- gradle/libs.versions.toml | 8 + .../schema/packets/kafkatorioPacket.kt | 11 +- modules/events-server/build.gradle.kts | 5 + .../processor/KafkatorioTopology.kt | 85 ++++++++- .../processor/admin/kafkatorioTopics.kt | 18 +- .../processor/config/ApplicationProperties.kt | 3 + .../kafkatorio/processor/eventProcessor.kt | 10 +- .../tileserver/SyslogSocketServer.kt | 168 ++++++++++++++++++ .../topology/factorioServerPackets.kt | 22 ++- .../src/main/resources/config.yml | 6 + .../src/docker-compose.yml | 13 +- .../adamko/kafkatorio/webmap/websockets.kt | 6 +- 13 files changed, 327 insertions(+), 30 deletions(-) create mode 100644 modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt diff --git a/build.gradle.kts b/build.gradle.kts index fd9e82d..ae6a2ff 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { `build-dashboard` // kotlin("multiplatform") apply false // try to stop error 'Projects must be configuring' - kotlin("js") apply false // try to stop error 'Projects must be configuring' +// kotlin("js") apply false // try to stop error 'Projects must be configuring' } group = "dev.adamko.kafkatorio" diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7cda9a6..9d174aa 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -35,6 +35,8 @@ ktor = "2.0.2" hoplite = "2.1.5" +simpleSyslogParser = "0.0.6" + kotkaStreams = "dev-SNAPSHOT" ## test ## @@ -156,10 +158,12 @@ ktor-serverCallLogging = { group = "io.ktor", name = "ktor-server-call-l ktor-serverCallId = { group = "io.ktor", name = "ktor-server-call-id" } ktor-network = { group = "io.ktor", name = "ktor-network" } +ktor-networkTls = { group = "io.ktor", name = "ktor-network-tls" } ktor-test-clientMock = { group = "io.ktor", name = "ktor-client-mock" } ktor-test-serverHost = { group = "io.ktor", name = "ktor-server-test-host" } + ### jackson ## #jackson-bom = { group = " com.fasterxml.jackson.module", name = ":jackson-module-kotlin", version = "2.13.+" } #jackson-kotlin = { group = " com.fasterxml.jackson.module", name = ":jackson-module-kotlin", version = "2.13.+" } @@ -205,6 +209,10 @@ hoplite-core = { group = "com.sksamuel.hoplite", name = "hoplite-core", hoplite-datetime = { group = "com.sksamuel.hoplite", name = "hoplite-datetime", version.ref = "hoplite" } hoplite-yaml = { group = "com.sksamuel.hoplite", name = "hoplite-yaml", version.ref = "hoplite" } + +simpleSyslogParser = { group = "com.github.palindromicity", name = "simple-syslog", version.ref = "simpleSyslogParser" } + + ############# ### test ### ############# diff --git a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt index e1d52fb..b3f2a53 100644 --- a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt +++ b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt @@ -16,4 +16,13 @@ data class KafkatorioPacket( @Serializable @SerialName("kafkatorio.packet.KafkatorioPacketData") -sealed class KafkatorioPacketData +sealed class KafkatorioPacketData { + + @Serializable + @SerialName("kafkatorio.packet.KafkatorioPacketData.Error") + data class Error( + val message: String? = null, + val rawValue: String? = null, + ) : KafkatorioPacketData() + +} diff --git a/modules/events-server/build.gradle.kts b/modules/events-server/build.gradle.kts index 7978403..58084d5 100644 --- a/modules/events-server/build.gradle.kts +++ b/modules/events-server/build.gradle.kts @@ -45,6 +45,11 @@ dependencies { implementation(libs.okio.core) + implementation(libs.ktor.network) + implementation(libs.ktor.networkTls) + + implementation(libs.simpleSyslogParser) + implementation(libs.kafka.kotkaStreams) { isChanging = true } diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt index 1aa3e53..d86e259 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt @@ -1,6 +1,9 @@ package dev.adamko.kafkatorio.processor +import com.github.palindromicity.syslog.dsl.SyslogFieldKeys +import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG import dev.adamko.kafkatorio.processor.config.ApplicationProperties +import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer import dev.adamko.kafkatorio.processor.topology.broadcastToWebsocket import dev.adamko.kafkatorio.processor.topology.colourMapChunks @@ -17,10 +20,17 @@ import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.cancel import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.job import kotlinx.coroutines.launch import kotlinx.coroutines.suspendCancellableCoroutine +import org.apache.kafka.clients.producer.KafkaProducer import org.apache.kafka.clients.producer.ProducerConfig +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.header.internals.RecordHeader +import org.apache.kafka.common.serialization.Serdes import org.apache.kafka.streams.KafkaStreams import org.apache.kafka.streams.StreamsBuilder import org.apache.kafka.streams.StreamsConfig @@ -29,12 +39,14 @@ import org.apache.kafka.streams.TopologyDescription internal class KafkatorioTopology( - private val websocketServer: WebsocketServer, private val appProps: ApplicationProperties, + private val websocketServer: WebsocketServer, + private val syslogServer: SyslogSocketServer, ) { private val builderScope = CoroutineScope(CoroutineName("KafkaTopology")) + suspend fun start() = builderScope.launch { splitPackets() @@ -42,8 +54,11 @@ internal class KafkatorioTopology( saveTiles() websocketBroadcaster() + + syslogProducer() } + private suspend fun splitPackets() { val builder = StreamsBuilder() val packets = factorioServerPacketStream(builder) @@ -54,18 +69,21 @@ internal class KafkatorioTopology( launchTopology("splitPackets", topology) } + private suspend fun groupTilesMapChunks() { val builder = StreamsBuilder() val topology = colourMapChunks(builder) launchTopology("groupTilesMapChunks", topology) } + private suspend fun saveTiles() { val builder = StreamsBuilder() val topology = saveMapTiles(builder, appProps.tileDir) launchTopology("saveTiles", topology) } + private suspend fun websocketBroadcaster() { val builder = StreamsBuilder() broadcastToWebsocket(websocketServer, builder) @@ -75,11 +93,65 @@ internal class KafkatorioTopology( launchTopology( "websocketBroadcaster", topology, mapOf( StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}", - StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${10.milliseconds.inWholeMilliseconds}" + StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}" ) ) } + + private fun syslogProducer( + id: String = "syslog-producer" + ) { + val props: Properties = appProps.kafkaStreamsConfig.toProperties() + + val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String + props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId) + + + val producer: KafkaProducer = KafkaProducer( + props, + Serdes.String().serializer(), + Serdes.String().serializer(), + ) + + builderScope.coroutineContext.job.invokeOnCompletion { + producer.close() + } + + syslogServer.messages + .filter { it.message?.startsWith("KafkatorioPacket:") == true } + .onEach { syslogMsg -> + + val kafkatorioMessage = syslogMsg.message?.substringAfter("KafkatorioPacket:") + + val record = ProducerRecord( + TOPIC_SRC_SERVER_LOG, + "syslog-test", + kafkatorioMessage, + ) + + val headers = SyslogFieldKeys.values() + .filter { "HEADER_" in it.name } + .mapNotNull { headerKey -> + val headerValue = syslogMsg.src[headerKey.field] + when { + headerValue.isNullOrBlank() -> null + else -> RecordHeader( + headerKey.field, + headerValue.toByteArray() + ) + } + } + + headers.forEach { header -> + record.headers().add(header) + } + + producer.send(record) + }.launchIn(builderScope) + } + + private suspend fun launchTopology( id: String, topology: Topology, @@ -105,11 +177,8 @@ internal class KafkatorioTopology( streams.setStateListener { newState: KafkaStreams.State, _ -> when (newState) { KafkaStreams.State.NOT_RUNNING, -// KafkaStreams.State.ERROR, - -> if (cont.isActive) { - cont.resume(newState) - } - else -> Unit // do nothing + KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState) + else -> Unit // do nothing } } @@ -127,6 +196,7 @@ internal class KafkatorioTopology( } } + private fun printTopologyDescription(appId: String, topology: Topology) { val description: TopologyDescription = topology.describe() @@ -146,6 +216,7 @@ internal class KafkatorioTopology( ) } + companion object { private fun Map.toProperties(): Properties = entries.fold(Properties()) { props, (k, v) -> diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt index 1bbfe9e..41c0206 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt @@ -48,14 +48,15 @@ fun allTopics(): Set = buildSet { val KafkatorioPacketData.topicName: String get() = when (this) { - is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class) - is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class) - is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class) - is PrototypesUpdate -> packetTopicNames.getValue(PrototypesUpdate::class) - is SurfaceUpdate -> packetTopicNames.getValue(SurfaceUpdate::class) - is EntityUpdate -> packetTopicNames.getValue(EntityUpdate::class) - is MapChunkUpdate -> packetTopicNames.getValue(MapChunkUpdate::class) - is PlayerUpdate -> packetTopicNames.getValue(PlayerUpdate::class) + is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class) + is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class) + is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class) + is PrototypesUpdate -> packetTopicNames.getValue(PrototypesUpdate::class) + is SurfaceUpdate -> packetTopicNames.getValue(SurfaceUpdate::class) + is EntityUpdate -> packetTopicNames.getValue(EntityUpdate::class) + is MapChunkUpdate -> packetTopicNames.getValue(MapChunkUpdate::class) + is PlayerUpdate -> packetTopicNames.getValue(PlayerUpdate::class) + is KafkatorioPacketData.Error -> packetTopicNames.getValue(KafkatorioPacketData.Error::class) } @@ -68,6 +69,7 @@ private val packetTopicNames: Map, String> = ma EntityUpdate::class to "entity", MapChunkUpdate::class to "map-chunk", PlayerUpdate::class to "player", + KafkatorioPacketData.Error::class to "error", ).mapValues { (_, v) -> "$DOMAIN.packet.$v" } diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt index 6881ecb..85ac987 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt @@ -12,6 +12,9 @@ data class ApplicationProperties( val tileDir: Path, val kafkaAdminConfig: Map, val kafkaStreamsConfig: Map, + + val socketServerHost: String, + val socketServerPort: Int, ) { companion object { diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt index 2515009..ae1e860 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt @@ -3,6 +3,7 @@ package dev.adamko.kafkatorio.processor import dev.adamko.kafkatorio.processor.admin.KafkatorioKafkaAdmin import dev.adamko.kafkatorio.processor.config.ApplicationProperties +import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer import dev.adamko.kafkatorio.processor.tileserver.WebMapTileServer import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer import dev.adamko.kafkatorio.processor.tileserver.webServer @@ -11,7 +12,6 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.job import kotlinx.coroutines.launch -import kotlinx.serialization.Serializable import org.http4k.server.Http4kServer @@ -22,12 +22,18 @@ suspend fun main() { val admin = KafkatorioKafkaAdmin(appProps) admin.createKafkatorioTopics() + + val syslogServer = SyslogSocketServer(appProps) val wsServer = WebsocketServer() val tileServer = WebMapTileServer(appProps) + val topology = KafkatorioTopology(appProps, wsServer, syslogServer) coroutineScope { launch { - val topology = KafkatorioTopology(wsServer, appProps) + syslogServer.start() + } + + launch { topology.start() println("launched KafkatorioTopology") diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt new file mode 100644 index 0000000..3bf3731 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt @@ -0,0 +1,168 @@ +package dev.adamko.kafkatorio.processor.tileserver + +import com.github.palindromicity.syslog.AllowableDeviations +import com.github.palindromicity.syslog.SyslogParser +import com.github.palindromicity.syslog.SyslogParserBuilder +import com.github.palindromicity.syslog.SyslogSpecification +import com.github.palindromicity.syslog.dsl.SyslogFieldKeys +import dev.adamko.kafkatorio.processor.config.ApplicationProperties +import dev.adamko.kafkatorio.processor.tileserver.SyslogMsg.Companion.toSyslogMsg +import io.ktor.network.selector.SelectorManager +import io.ktor.network.sockets.aSocket +import io.ktor.network.sockets.awaitClosed +import io.ktor.network.sockets.openReadChannel +import io.ktor.utils.io.readUTF8Line +import java.util.EnumSet +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.yield + + +class SyslogSocketServer( + appProps: ApplicationProperties, + dispatcher: CoroutineDispatcher = Dispatchers.IO, +) { + + private val selectorManager: SelectorManager = SelectorManager(dispatcher) + + + private val serverSocketTcp = aSocket(selectorManager) + .tcp() + .bind( + appProps.socketServerHost, + appProps.socketServerPortTcp, + ) + + +// private val serverSocketUdp: BoundDatagramSocket = aSocket(selectorManager) +// .udp() +// .bind( +// localAddress = InetSocketAddress( +// appProps.socketServerHost, +// appProps.socketServerPortUdp, +// ), +// ) + + + private val syslogParser: SyslogParser = SyslogParserBuilder() + .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION)) + .forSpecification(SyslogSpecification.RFC_5424) + .build() + + + private val _messages: MutableSharedFlow = MutableSharedFlow() + val messages: SharedFlow + get() = _messages.asSharedFlow() + + + suspend fun start() { +// udp() + tcp() + } + +// private suspend fun udp() { +// println("[SyslogSocketServer] started listening: ${serverSocketUdp.localAddress}") +// +// serverSocketUdp.incoming +// .receiveAsFlow() +// .map {udpPacket: Datagram -> +// val msg = udpPacket.packet.readText() +// println("[SyslogSocketServer] received msg: $msg") +// syslogParser.parseLine(msg).toSyslogMsg() +// }.collect(_messages) +// } + + private suspend fun tcp() = coroutineScope { + println("[SyslogSocketServer] started listening: ${serverSocketTcp.localAddress}") + + while (isActive) { + val socket = serverSocketTcp.accept() + + launch { + val channel = socket.openReadChannel() + try { + while (socket.socketContext.isActive) { + val line = channel.readUTF8Line() + println("[SyslogSocketServer] received: $line") + + val syslog = syslogParser.parseLine(line).toSyslogMsg() + + _messages.emit(syslog) + + yield() + } + } catch (e: Exception) { + socket.awaitClosed() + } + } + } + + yield() + } +} + + +data class SyslogMsg( + val src: Map +) { + + val message: String? + get() = src[SyslogFieldKeys.MESSAGE.field] + + val headerAppName: String? + get() = src[SyslogFieldKeys.HEADER_APPNAME.field] + + val headerHostName: String? + get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field] + + val headerPri: String? + get() = src[SyslogFieldKeys.HEADER_PRI.field] + + val headerPriSeverity: String? + get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field] + + val headerPriFacility: String? + get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field] + + val headerProcId: String? + get() = src[SyslogFieldKeys.HEADER_PROCID.field] + + val headerTimestamp: String? + get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field] + + val headerMsgId: String? + get() = src[SyslogFieldKeys.HEADER_MSGID.field] + + val headerVersion: String? + get() = src[SyslogFieldKeys.HEADER_VERSION.field] + + val structuredBase: String? + get() = src[SyslogFieldKeys.STRUCTURED_BASE.field] + + val structuredElementIdFmt: String? + get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field] + + val structuredElementIdPnameFmt: String? + get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field] + + val structuredElementIdPnamePattern: String? + get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field] + + companion object { + fun Map.toSyslogMsg(): SyslogMsg { + val map = entries.mapNotNull { (k, v) -> + when (v) { + is String -> k to v + else -> null + } + }.toMap() + return SyslogMsg(map) + } + } +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt index f19cfc5..283f062 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt @@ -5,7 +5,9 @@ import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG import dev.adamko.kafkatorio.processor.admin.topicName import dev.adamko.kafkatorio.processor.serdes.jsonMapper import dev.adamko.kafkatorio.schema.common.FactorioServerId +import dev.adamko.kafkatorio.schema.common.Tick import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket +import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData import dev.adamko.kotka.extensions.component1 import dev.adamko.kotka.extensions.component2 import dev.adamko.kotka.extensions.consumedAs @@ -31,8 +33,26 @@ fun factorioServerPacketStream( Serdes.String(), ) ).map("decode-packets") { serverId: String, value: String -> - val packet = jsonMapper.decodeFromString(value) + val packet = runCatching { + jsonMapper.decodeFromString(value) + }.getOrElse { e -> + KafkatorioPacket( + modVersion = "unknown", + tick = Tick(0u), + data = KafkatorioPacketData.Error( + message = e.message, + rawValue = value, + ) + ) + } + + if (packet.data is KafkatorioPacketData.Error) { + println("error parsing $TOPIC_SRC_SERVER_LOG message: ${packet.data}") + } + FactorioServerId(serverId) to packet + }.filter { _, value -> + value.data !is KafkatorioPacketData.Error } } diff --git a/modules/events-server/src/main/resources/config.yml b/modules/events-server/src/main/resources/config.yml index fa31386..9b8eb5c 100644 --- a/modules/events-server/src/main/resources/config.yml +++ b/modules/events-server/src/main/resources/config.yml @@ -3,10 +3,16 @@ web-port: ${SERVER_PORT:-12080} tile-dir: ${TILE_DIR:-./tile-dir} + +socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1} +socket-server-port: ${SOCKET_SERVER_PORT:-9002} + + kafka-admin-config: client.id: kafkatorio.events-server-admin bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092} + kafka-streams-config: application.id: kafkatorio-events-processor bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092} diff --git a/modules/infra-factorio-server/src/docker-compose.yml b/modules/infra-factorio-server/src/docker-compose.yml index 262ca06..3b12252 100644 --- a/modules/infra-factorio-server/src/docker-compose.yml +++ b/modules/infra-factorio-server/src/docker-compose.yml @@ -4,8 +4,6 @@ services: factorio-server: image: factoriotools/factorio:${FACTORIO_VERSION} - # logging: - # driver: "json-file" container_name: "factorio-server" restart: "no" networks: @@ -18,12 +16,11 @@ services: - "27015:27015/tcp" # rcon volumes: - "./factorio-server:/factorio" - # logging: - # driver: "mickyg/kafka-logdriver:latest" - # options: - # KAFKA_BROKER_ADDR: "kafka-kraft" - # LOG_TOPIC: "dockerlogs-factorio-server" - # LOG_TAG: "factorio-server" + logging: + driver: syslog + options: + syslog-address: "tcp://host.docker.internal:9002" + syslog-format: "rfc5424" # environment: # RCON_PORT: 27715 diff --git a/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt b/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt index 7c0ca27..077472f 100644 --- a/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt +++ b/modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt @@ -5,6 +5,7 @@ import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate import dev.adamko.kafkatorio.schema.packets.EntityUpdate import dev.adamko.kafkatorio.schema.packets.EventServerPacket +import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate import dev.adamko.kafkatorio.schema.packets.PlayerUpdate import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate @@ -74,7 +75,7 @@ class WebsocketService( is EventServerPacket.Kafkatorio -> when (val packetData = packet.packet.data) { - is PlayerUpdate -> reduxStore.dispatch( + is PlayerUpdate -> reduxStore.dispatch( FactorioUpdate.Player(packet.packet.tick, packetData) ) is ConfigurationUpdate, @@ -83,7 +84,8 @@ class WebsocketService( is PrototypesUpdate, is SurfaceUpdate, is EntityUpdate, - is MapChunkUpdate -> { + is MapChunkUpdate, + is KafkatorioPacketData.Error -> { // to be continued... } } From fc758030fd7c26e6f2f31380a17a159c21afd412 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:08:39 +0200 Subject: [PATCH 17/22] event-server tidy up (improve value class, rm unused class, rename websocket broadcaster) --- .../kafkatorio/processor/topology/colourMapChunks.kt | 2 +- .../kafkatorio/processor/topology/groupMapChunks.kt | 1 - .../adamko/kafkatorio/processor/topology/prototypes.kt | 8 +++++--- .../topology/{playerUpdates.kt => websocketBroadcast.kt} | 0 4 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt rename modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/{playerUpdates.kt => websocketBroadcast.kt} (100%) diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt index c471d18..b790ffc 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt @@ -187,7 +187,7 @@ private fun KTable>.enr val missingProtos = mutableSetOf() val tileColours = tiles.map.mapValues { (_, code) -> - colourDict.map.getOrElse(code) { + colourDict.getOrElse(code) { missingProtos.add(code) ColourHex.TRANSPARENT } diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt deleted file mode 100644 index 272d386..0000000 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt +++ /dev/null @@ -1 +0,0 @@ -package dev.adamko.kafkatorio.processor.topology diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt index 56b9e7b..85401e5 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt @@ -29,7 +29,9 @@ value class TileProtoHashCode private constructor(val code: Int) { @JvmInline @Serializable -value class TileColourDict(val map: Map) +value class TileColourDict( + private val map: Map +) : Map by map /** Get the latest map-tile colours per server. */ @@ -48,9 +50,9 @@ fun KStream.createTilePrototypeTable() TileColourDict(map) }.filter("$pid.filterMapTileProtos") { _, dict: TileColourDict -> - dict.map.isNotEmpty() + dict.isNotEmpty() }.peek { serverId, dict: TileColourDict -> - println("$pid: server $serverId has TileColourDict[${dict.map.size}]: ${dict.map.entries.joinToString()}") + println("$pid: server $serverId has TileColourDict[${dict.size}]: ${dict.entries.joinToString()}") }.repartition( repartitionedAs( "$pid.pre-table-repartition", diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/playerUpdates.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/websocketBroadcast.kt similarity index 100% rename from modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/playerUpdates.kt rename to modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/websocketBroadcast.kt From a2e8ab61d7ca732224bdaebd74b32b6fc0aefd70 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:08:58 +0200 Subject: [PATCH 18/22] change default port to 8080 (works better with Traefik defaults) --- modules/events-server/src/main/resources/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/events-server/src/main/resources/config.yml b/modules/events-server/src/main/resources/config.yml index 9b8eb5c..0e12ff8 100644 --- a/modules/events-server/src/main/resources/config.yml +++ b/modules/events-server/src/main/resources/config.yml @@ -1,6 +1,6 @@ # Factorio Events Processor config -web-port: ${SERVER_PORT:-12080} +web-port: ${SERVER_PORT:-8080} tile-dir: ${TILE_DIR:-./tile-dir} From 5f1187f9a8ec59c8817264e0ca3781a8b50cbb4f Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:09:48 +0200 Subject: [PATCH 19/22] rename: socketServerPortTcp -> socketServerPort --- .../kafkatorio/processor/tileserver/SyslogSocketServer.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt index 3bf3731..4a1da1c 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt @@ -36,7 +36,7 @@ class SyslogSocketServer( .tcp() .bind( appProps.socketServerHost, - appProps.socketServerPortTcp, + appProps.socketServerPort, ) From 1d0b2b8eaa5b273478db504794b6b89c79b55db9 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:57:57 +0200 Subject: [PATCH 20/22] refactor typescript files (mostly moving&renaming for consistency) --- modules/events-mod/package.json | 12 +++-- .../events-mod/src/main/typescript/control.ts | 20 ++------ .../kafkatorio-schema.ts | 8 +++ .../src/main/typescript/main/PacketEmitter.ts | 3 +- .../typescript/main/cache/cacheChecker.ts | 10 ---- .../main/commands/kafkatorioCommands.ts | 9 ++-- .../configurationUpdate.ts} | 18 ++++++- ...soleEventHandlers.ts => consoleUpdates.ts} | 4 +- .../main/typescript/main/events/converters.ts | 11 +++- .../{EntityEvents.ts => entityUpdates.ts} | 2 +- ...nkEventListeners.ts => mapChunkUpdates.ts} | 2 +- ...ayerEventListeners.ts => playerUpdates.ts} | 2 +- .../prototypeUpdates.ts} | 26 +++------- .../main/events/registerHandlers.ts | 50 +++++-------------- .../typescript/main/events/settingsUpdate.ts | 7 +++ .../events/{handlers.ts => surfaceUpdates.ts} | 2 +- .../main/settings/KafkatorioSettings.ts | 23 --------- .../src/main/typescript/main/types.d.ts | 2 +- .../infra-kafka-pipe/src/docker-compose.yml | 6 +-- 19 files changed, 91 insertions(+), 126 deletions(-) rename modules/events-mod/src/main/typescript/generated/{kafkatorio-schema => }/kafkatorio-schema.ts (96%) delete mode 100644 modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts rename modules/events-mod/src/main/typescript/main/{config-update/handleConfigurationUpdate.ts => events/configurationUpdate.ts} (75%) rename modules/events-mod/src/main/typescript/main/events/{onConsoleEventHandlers.ts => consoleUpdates.ts} (97%) rename modules/events-mod/src/main/typescript/main/events/{EntityEvents.ts => entityUpdates.ts} (97%) rename modules/events-mod/src/main/typescript/main/events/{onMapChunkEventListeners.ts => mapChunkUpdates.ts} (99%) rename modules/events-mod/src/main/typescript/main/events/{onPlayerEventListeners.ts => playerUpdates.ts} (98%) rename modules/events-mod/src/main/typescript/main/{config-update/prototypes.ts => events/prototypeUpdates.ts} (70%) create mode 100644 modules/events-mod/src/main/typescript/main/events/settingsUpdate.ts rename modules/events-mod/src/main/typescript/main/events/{handlers.ts => surfaceUpdates.ts} (94%) diff --git a/modules/events-mod/package.json b/modules/events-mod/package.json index 4b3a926..50840cf 100644 --- a/modules/events-mod/package.json +++ b/modules/events-mod/package.json @@ -9,7 +9,8 @@ "engines": { "node": "~14.19.1" }, - "devDependencies": {}, + "devDependencies": { + }, "dependencies": { "lua-types": "^2.11.0", "typescript-to-lua": "1.6.0", @@ -17,7 +18,10 @@ "typescript": "4.7.4" }, "license": "UNLICENSED", - "peerDependencies": {}, - "optionalDependencies": {}, - "bundledDependencies": [] + "peerDependencies": { + }, + "optionalDependencies": { + }, + "bundledDependencies": [ + ] } diff --git a/modules/events-mod/src/main/typescript/control.ts b/modules/events-mod/src/main/typescript/control.ts index abf359c..3800be5 100644 --- a/modules/events-mod/src/main/typescript/control.ts +++ b/modules/events-mod/src/main/typescript/control.ts @@ -1,23 +1,11 @@ -import {emitConfigurationUpdate} from "./main/config-update/handleConfigurationUpdate"; -import {emitPrototypes} from "./main/config-update/prototypes"; import {initGlobal} from "./main/global-init"; -import KafkatorioSettings from "./main/settings/KafkatorioSettings"; +import {emitPrototypes} from "./main/events/prototypeUpdates"; require("./main/events/registerHandlers") -require("./main/events/onMapChunkEventListeners") -require("./main/events/onPlayerEventListeners") -require("./main/events/EntityEvents") +require("./main/events/mapChunkUpdates") +require("./main/events/playerUpdates") +require("./main/events/entityUpdates") require("./main/commands/kafkatorioCommands") -require("./main/cache/cacheChecker") - - -script.on_configuration_changed((data: ConfigurationChangedData) => { - // runs whenever the version of the base game, or a mod, changes - initGlobal(true) - // KafkatorioSettings.loadSettings() - emitConfigurationUpdate(data) - emitPrototypes() -}) script.on_init(() => { diff --git a/modules/events-mod/src/main/typescript/generated/kafkatorio-schema/kafkatorio-schema.ts b/modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts similarity index 96% rename from modules/events-mod/src/main/typescript/generated/kafkatorio-schema/kafkatorio-schema.ts rename to modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts index ae19634..f0559a9 100644 --- a/modules/events-mod/src/main/typescript/generated/kafkatorio-schema/kafkatorio-schema.ts +++ b/modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts @@ -13,6 +13,7 @@ export type KafkatorioPacketData = | KafkatorioPacketData.ConsoleChatUpdate | KafkatorioPacketData.ConsoleCommandUpdate | KafkatorioPacketData.EntityUpdate + | KafkatorioPacketData.Error | KafkatorioPacketData.MapChunkUpdate | KafkatorioPacketData.PlayerUpdate | KafkatorioPacketData.PrototypesUpdate @@ -28,6 +29,7 @@ export namespace KafkatorioPacketData { EntityUpdate = "kafkatorio.packet.keyed.EntityUpdate", MapChunkUpdate = "kafkatorio.packet.keyed.MapChunkUpdate", PlayerUpdate = "kafkatorio.packet.keyed.PlayerUpdate", + Error = "kafkatorio.packet.KafkatorioPacketData.Error", } export interface ConfigurationUpdate { @@ -115,6 +117,12 @@ export namespace KafkatorioPacketData { disconnectReason?: string | null; isRemoved?: boolean | null; } + + export interface Error { + type: KafkatorioPacketData.Type.Error; + message?: string | null; + rawValue?: string | null; + } } export type UInt = uint; diff --git a/modules/events-mod/src/main/typescript/main/PacketEmitter.ts b/modules/events-mod/src/main/typescript/main/PacketEmitter.ts index 55a1ba7..a566499 100644 --- a/modules/events-mod/src/main/typescript/main/PacketEmitter.ts +++ b/modules/events-mod/src/main/typescript/main/PacketEmitter.ts @@ -1,9 +1,10 @@ import { KafkatorioPacket, KafkatorioPacketData -} from "../generated/kafkatorio-schema/kafkatorio-schema"; +} from "../generated/kafkatorio-schema"; import {KafkatorioKeyedPacketData} from "./types"; + export class KafkatorioPacketEmitter { emitInstantPacket( diff --git a/modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts b/modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts deleted file mode 100644 index face16a..0000000 --- a/modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts +++ /dev/null @@ -1,10 +0,0 @@ -import EventCacheService from "./EventDataCache"; - - -/** Every 0.1 seconds */ -script.on_nth_tick(6, (nthTick: NthTickEventData) => { - const emittedCount = EventCacheService.extractAndEmitExpiredPackets() - if (emittedCount > 0) { - log(`[on_nth_tick:${nthTick.tick}] emitted ${emittedCount} events`) - } -}) diff --git a/modules/events-mod/src/main/typescript/main/commands/kafkatorioCommands.ts b/modules/events-mod/src/main/typescript/main/commands/kafkatorioCommands.ts index 19a34c5..c9bdcc0 100644 --- a/modules/events-mod/src/main/typescript/main/commands/kafkatorioCommands.ts +++ b/modules/events-mod/src/main/typescript/main/commands/kafkatorioCommands.ts @@ -1,7 +1,6 @@ -import {emitPrototypes} from "../config-update/prototypes"; import {initGlobal} from "../global-init"; -import floor = math.floor; import {Queue} from "../queue/queue"; +import {emitPrototypes} from "../events/prototypeUpdates"; commands.add_command( @@ -32,8 +31,8 @@ commands.add_command( let player = game.players[e.player_index] const chunkPosition: ChunkPosition = { - x: floor(player.position.x / 32), - y: floor(player.position.y / 32), + x: math.floor(player.position.x / 32), + y: math.floor(player.position.y / 32), } const chunkXMin = chunkPosition.x - radius @@ -56,7 +55,7 @@ commands.add_command( position: {x: chunk.x, y: chunk.y}, area: chunk.area, surface: surface, - tick: e.tick + tick: e.tick, } Queue.enqueue( diff --git a/modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts b/modules/events-mod/src/main/typescript/main/events/configurationUpdate.ts similarity index 75% rename from modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts rename to modules/events-mod/src/main/typescript/main/events/configurationUpdate.ts index 4808d6c..b11863f 100644 --- a/modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts +++ b/modules/events-mod/src/main/typescript/main/events/configurationUpdate.ts @@ -3,11 +3,23 @@ import { ConfigurationUpdateGameData, ConfigurationUpdateModData, KafkatorioPacketData -} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +} from "../../generated/kafkatorio-schema"; +import {initGlobal} from "../global-init"; +import {emitPrototypes} from "./prototypeUpdates"; import Type = KafkatorioPacketData.Type; import ConfigurationUpdate = KafkatorioPacketData.ConfigurationUpdate; -export function emitConfigurationUpdate(changeData: ConfigurationChangedData) { + +script.on_configuration_changed((data: ConfigurationChangedData) => { + // runs whenever the version of the base game, or a mod, changes + initGlobal(true) + // KafkatorioSettings.loadSettings() + emitConfigurationUpdate(data) + emitPrototypes() +}) + + +function emitConfigurationUpdate(changeData: ConfigurationChangedData) { const configUpdateData: ConfigurationUpdate = { type: Type.ConfigurationUpdate, @@ -20,6 +32,7 @@ export function emitConfigurationUpdate(changeData: ConfigurationChangedData) { PacketEmitter.emitInstantPacket(configUpdateData) } + function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdateGameData { return { oldVersion: changeData.old_version ?? null, @@ -27,6 +40,7 @@ function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdate } } + function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] { let namesOfChangedMods: string[] = Object.keys(e.mod_changes) diff --git a/modules/events-mod/src/main/typescript/main/events/onConsoleEventHandlers.ts b/modules/events-mod/src/main/typescript/main/events/consoleUpdates.ts similarity index 97% rename from modules/events-mod/src/main/typescript/main/events/onConsoleEventHandlers.ts rename to modules/events-mod/src/main/typescript/main/events/consoleUpdates.ts index c76b673..08b2c48 100644 --- a/modules/events-mod/src/main/typescript/main/events/onConsoleEventHandlers.ts +++ b/modules/events-mod/src/main/typescript/main/events/consoleUpdates.ts @@ -1,7 +1,8 @@ import PacketEmitter from "../PacketEmitter"; -import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +import {KafkatorioPacketData} from "../../generated/kafkatorio-schema"; import Type = KafkatorioPacketData.Type; + script.on_event( defines.events.on_console_chat, (event: OnConsoleChatEvent) => { @@ -15,6 +16,7 @@ script.on_event( } ) + script.on_event( defines.events.on_console_command, (event: OnConsoleCommandEvent) => { diff --git a/modules/events-mod/src/main/typescript/main/events/converters.ts b/modules/events-mod/src/main/typescript/main/events/converters.ts index 81da5f1..db6fe3e 100644 --- a/modules/events-mod/src/main/typescript/main/events/converters.ts +++ b/modules/events-mod/src/main/typescript/main/events/converters.ts @@ -1,5 +1,5 @@ import {EventName} from "../types"; -import {Colour, MapChunkPosition} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +import {Colour, MapChunkPosition} from "../../generated/kafkatorio-schema"; export namespace Converters { @@ -57,4 +57,13 @@ export namespace Converters { return [position.x, position.y] } + + export function convertCollisionMaskToNames(cm: CollisionMask): string[] { + let masks: string[] = [] + for (let [name] of pairs(cm)) { + masks.push(name) + } + return masks + } + } diff --git a/modules/events-mod/src/main/typescript/main/events/EntityEvents.ts b/modules/events-mod/src/main/typescript/main/events/entityUpdates.ts similarity index 97% rename from modules/events-mod/src/main/typescript/main/events/EntityEvents.ts rename to modules/events-mod/src/main/typescript/main/events/entityUpdates.ts index 34c2ff3..791f3e6 100644 --- a/modules/events-mod/src/main/typescript/main/events/EntityEvents.ts +++ b/modules/events-mod/src/main/typescript/main/events/entityUpdates.ts @@ -3,7 +3,7 @@ import EventDataCache from "../cache/EventDataCache"; import { EntityUpdateKey, KafkatorioPacketData -} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +} from "../../generated/kafkatorio-schema"; script.on_event(defines.events.on_built_entity, (e: OnBuiltEntityEvent) => { diff --git a/modules/events-mod/src/main/typescript/main/events/onMapChunkEventListeners.ts b/modules/events-mod/src/main/typescript/main/events/mapChunkUpdates.ts similarity index 99% rename from modules/events-mod/src/main/typescript/main/events/onMapChunkEventListeners.ts rename to modules/events-mod/src/main/typescript/main/events/mapChunkUpdates.ts index 858772a..509afc5 100644 --- a/modules/events-mod/src/main/typescript/main/events/onMapChunkEventListeners.ts +++ b/modules/events-mod/src/main/typescript/main/events/mapChunkUpdates.ts @@ -4,7 +4,7 @@ import EventUpdatesManager, {EventUpdates} from "../cache/EventDataCache"; import { KafkatorioPacketData, MapChunkPosition -} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +} from "../../generated/kafkatorio-schema"; import CacheKey = EventUpdates.PacketKey; import Type = KafkatorioPacketData.Type; diff --git a/modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts b/modules/events-mod/src/main/typescript/main/events/playerUpdates.ts similarity index 98% rename from modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts rename to modules/events-mod/src/main/typescript/main/events/playerUpdates.ts index 22ad180..da55741 100644 --- a/modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts +++ b/modules/events-mod/src/main/typescript/main/events/playerUpdates.ts @@ -3,7 +3,7 @@ import EventUpdatesManager from "../cache/EventDataCache"; import { KafkatorioPacketData, PlayerUpdateKey -} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +} from "../../generated/kafkatorio-schema"; import Type = KafkatorioPacketData.Type; import packetEmitter from "../PacketEmitter"; diff --git a/modules/events-mod/src/main/typescript/main/config-update/prototypes.ts b/modules/events-mod/src/main/typescript/main/events/prototypeUpdates.ts similarity index 70% rename from modules/events-mod/src/main/typescript/main/config-update/prototypes.ts rename to modules/events-mod/src/main/typescript/main/events/prototypeUpdates.ts index 85819cd..8c71f9b 100644 --- a/modules/events-mod/src/main/typescript/main/config-update/prototypes.ts +++ b/modules/events-mod/src/main/typescript/main/events/prototypeUpdates.ts @@ -1,24 +1,21 @@ -import {Converters} from "../events/converters"; import PacketEmitter from "../PacketEmitter"; -import { - FactorioPrototype, - KafkatorioPacketData -} from "../../generated/kafkatorio-schema/kafkatorio-schema"; -import Type = KafkatorioPacketData.Type; +import {FactorioPrototype, KafkatorioPacketData} from "../../generated/kafkatorio-schema"; +import {Converters} from "./converters"; import PrototypesUpdate = KafkatorioPacketData.PrototypesUpdate; export function emitPrototypes() { const data: PrototypesUpdate = { - prototypes: prototypes(), - type: Type.PrototypesUpdate, + prototypes: prototypeUpdates(), + type: KafkatorioPacketData.Type.PrototypesUpdate, } PacketEmitter.emitInstantPacket(data) } -function prototypes(): FactorioPrototype[] { + +function prototypeUpdates(): FactorioPrototype[] { let prototypes: FactorioPrototype[] = [] prototypes.push(...getMapTilePrototypes()) @@ -26,6 +23,7 @@ function prototypes(): FactorioPrototype[] { return prototypes } + function getMapTilePrototypes(): FactorioPrototype.MapTile[] { let tiles: FactorioPrototype.MapTile[] = [] for (let [, tile] of game.tile_prototypes) { @@ -35,7 +33,7 @@ function getMapTilePrototypes(): FactorioPrototype.MapTile[] { name: tile.name, order: tile.order, layer: tile.layer, - collisionMasks: convertCollisionMaskToNames(tile.collision_mask), + collisionMasks: Converters.convertCollisionMaskToNames(tile.collision_mask), mapColour: Converters.mapColour(tile.map_color), canBeMined: tile.mineable_properties.minable, }) @@ -60,11 +58,3 @@ function getMapTilePrototypes(): FactorioPrototype.MapTile[] { // } // return tiles // } - -function convertCollisionMaskToNames(cm: CollisionMask): string[] { - let masks: string[] = [] - for (let [name] of pairs(cm)) { - masks.push(name) - } - return masks -} diff --git a/modules/events-mod/src/main/typescript/main/events/registerHandlers.ts b/modules/events-mod/src/main/typescript/main/events/registerHandlers.ts index f8b2aa7..d3758c7 100644 --- a/modules/events-mod/src/main/typescript/main/events/registerHandlers.ts +++ b/modules/events-mod/src/main/typescript/main/events/registerHandlers.ts @@ -1,34 +1,11 @@ import {Queue} from "../queue/queue"; import {isEventType} from "./eventTypeCheck"; import {Converters} from "./converters"; -import KafkatorioSettings from "../settings/KafkatorioSettings"; -import {handleChunkGeneratedEvent} from "./onMapChunkEventListeners"; -import {handleSurfaceUpdate} from "./handlers"; +import {handleChunkGeneratedEvent} from "./mapChunkUpdates"; +import {handleSurfaceUpdate} from "./surfaceUpdates"; +import EventCacheService from "../cache/EventDataCache"; -// script.on_event( -// defines.events.on_player_mined_entity, -// (e: OnPlayerMinedEntityEvent) => { -// let eventName = Converters.eventNameString(e.name) -// // handlePlayerUpdate(e.tick, eventName, e.player_index) -// handleEntityUpdate(e.tick, eventName, e.entity) -// } -// ) - -// script.on_event( -// defines.events.on_player_joined_game, -// (e: OnPlayerJoinedGameEvent) => { -// handlePlayerUpdate(e.tick, mapEventIdToName.get(e.name), e.player_index) -// } -// ) - -// script.on_event( -// defines.events.on_player_changed_position, -// (e: OnPlayerChangedPositionEvent) => { -// handlePlayerUpdate(e.tick, mapEventIdToName.get(e.name), e.player_index) -// } -// ) - script.on_event( defines.events.on_tick, (event: OnTickEvent) => { @@ -38,18 +15,26 @@ script.on_event( } } + if (event.tick % 6 == 0) { + // Every 0.1 seconds + const emittedCount = EventCacheService.extractAndEmitExpiredPackets() + if (emittedCount > 0) { + log(`[on_tick:${event.tick}] emitted ${emittedCount} events`) + } + } + if (event.tick % 7 == 0) { const events: EventData[] = Queue.dequeueValues(1) if (events.length > 0) { - log(`[${event.tick}] dequed ${events.length} events, current size: ${Queue.size()}`) + log(`[on_tick:${event.tick}] dequeued ${events.length} events, current size: ${Queue.size()}`) let i = 1 for (const event of events) { if (isEventType(event, defines.events.on_chunk_generated)) { let eName = Converters.eventNameString(event.name) - log(`[${event.tick}] dequed event ${eName}, delay ${i}`) + log(`[on_tick:${event.tick}] dequed event ${eName}, delay ${i}`) handleChunkGeneratedEvent(event, i) i += 1 } @@ -58,12 +43,3 @@ script.on_event( } } ) - - -script.on_event( - defines.events.on_runtime_mod_setting_changed, - (e: OnRuntimeModSettingChangedEvent) => { - print(`Settings changed ${e.setting_type}, ${e.player_index}, ${e.setting}, ${e.mod_name}`) - // KafkatorioSettings.loadSettings() - } -) diff --git a/modules/events-mod/src/main/typescript/main/events/settingsUpdate.ts b/modules/events-mod/src/main/typescript/main/events/settingsUpdate.ts new file mode 100644 index 0000000..93e3593 --- /dev/null +++ b/modules/events-mod/src/main/typescript/main/events/settingsUpdate.ts @@ -0,0 +1,7 @@ +script.on_event( + defines.events.on_runtime_mod_setting_changed, + (e: OnRuntimeModSettingChangedEvent) => { + print(`Settings changed ${e.setting_type}, ${e.player_index}, ${e.setting}, ${e.mod_name}`) + // KafkatorioSettings.loadSettings() + } +) diff --git a/modules/events-mod/src/main/typescript/main/events/handlers.ts b/modules/events-mod/src/main/typescript/main/events/surfaceUpdates.ts similarity index 94% rename from modules/events-mod/src/main/typescript/main/events/handlers.ts rename to modules/events-mod/src/main/typescript/main/events/surfaceUpdates.ts index 62989b8..9c90da5 100644 --- a/modules/events-mod/src/main/typescript/main/events/handlers.ts +++ b/modules/events-mod/src/main/typescript/main/events/surfaceUpdates.ts @@ -1,5 +1,5 @@ import PacketEmitter from "../PacketEmitter"; -import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema"; +import {KafkatorioPacketData} from "../../generated/kafkatorio-schema"; import Type = KafkatorioPacketData.Type; diff --git a/modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts b/modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts index bb90bfb..618554a 100644 --- a/modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts +++ b/modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts @@ -11,35 +11,12 @@ export class KafkatorioSettingsConfig { } - // private eventCacheExpirationTicks: Record = { - // "kafkatorio.packet.keyed.PlayerUpdate": - // KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.PlayerUpdate], - // "kafkatorio.packet.keyed.MapChunkUpdate": - // KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.MapChunkUpdate], - // "kafkatorio.packet.keyed.EntityUpdate": - // KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.EntityUpdate], - // } - public getEventCacheExpirationTicks(packet: KafkatorioKeyedPacketData): uint { const settingName = KafkatorioSettingsConfig.cacheDurationSettingName(packet.type) const seconds: double = settings.global[settingName].value as double return seconds * 60 // convert ticks to seconds } - // private static types: KafkatorioKeyedPacketTypes[] = [ - // KafkatorioPacketData.Type.PlayerUpdate, - // KafkatorioPacketData.Type.MapChunkUpdate, - // KafkatorioPacketData.Type.EntityUpdate, - // ] - - // public loadSettings(): void { - // for (const type of KafkatorioSettingsConfig.types) { - // const settingName = KafkatorioSettingsConfig.cacheDurationSettingName(type) - // const seconds: double = settings.global[settingName].value as double - // this.eventCacheExpirationTicks[type] = seconds * 60 // convert ticks to seconds - // } - // } - public initialiseSettings(data: Data): void { for (const [type,] of pairs(KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds)) { diff --git a/modules/events-mod/src/main/typescript/main/types.d.ts b/modules/events-mod/src/main/typescript/main/types.d.ts index 7505b54..fcdc914 100644 --- a/modules/events-mod/src/main/typescript/main/types.d.ts +++ b/modules/events-mod/src/main/typescript/main/types.d.ts @@ -1,4 +1,4 @@ -import {KafkatorioPacketData} from "../generated/kafkatorio-schema/kafkatorio-schema"; +import {KafkatorioPacketData} from "../generated/kafkatorio-schema"; export type EventName = keyof typeof defines.events diff --git a/modules/infra-kafka-pipe/src/docker-compose.yml b/modules/infra-kafka-pipe/src/docker-compose.yml index 4536152..dd90da2 100644 --- a/modules/infra-kafka-pipe/src/docker-compose.yml +++ b/modules/infra-kafka-pipe/src/docker-compose.yml @@ -20,10 +20,10 @@ services: KAFKA_HOST: "kafka:29092" volumes: # Windows (requires privileged/root) -# - "//var/run/docker.sock:/var/run/docker.sock" - + - "//var/run/docker.sock:/var/run/docker.sock" + # Rootless: - - "/run/user/1000/docker.sock:/var/run/docker.sock" +# - "/run/user/1000/docker.sock:/var/run/docker.sock" # Unix? # - "/var/run/docker.sock:/var/run/docker.sock" From 23a2ecb0d446aa92509a8f461e86f7334ab9b758 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:59:52 +0200 Subject: [PATCH 21/22] set serial name for MinedProduct and MinedProduct.Type --- .../src/main/typescript/generated/kafkatorio-schema.ts | 4 ++-- .../kotlin/dev/adamko/kafkatorio/schema/common/resources.kt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts b/modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts index f0559a9..e01e826 100644 --- a/modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts +++ b/modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts @@ -248,12 +248,12 @@ export type List = any; export type PrototypeKey = Int; export interface MinedProduct { - type: Type; + type: MinedProductType; name: PrototypeName; amount: Double | null; } -export enum Type { +export enum MinedProductType { item = "item", fluid = "fluid", } \ No newline at end of file diff --git a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt index d637494..d21a750 100644 --- a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt +++ b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt @@ -5,6 +5,7 @@ import kotlinx.serialization.Serializable @Serializable +@SerialName("kafkatorio.resource.MinedProduct") data class MinedProduct( /** `item` or `fluid`. */ val type: Type, @@ -30,6 +31,7 @@ data class MinedProduct( ) { @Serializable + @SerialName("kafkatorio.resource.MinedProductType") enum class Type { @SerialName("item") ITEM, From 68adff29a85629945a9de9528914243d1e8b20c9 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Jun 2022 23:00:11 +0200 Subject: [PATCH 22/22] try to diagnose weird getProjectTokens() null null null bug --- modules/events-mod/build.gradle.kts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/events-mod/build.gradle.kts b/modules/events-mod/build.gradle.kts index 7feaa3c..1f9084d 100644 --- a/modules/events-mod/build.gradle.kts +++ b/modules/events-mod/build.gradle.kts @@ -39,7 +39,7 @@ val projectTokens: MapProperty = ?: error("error getting projectTokens") //val projectTokens: MapProperty by rootProject.extra -val projectTokensX = projectTokens.apply { +val projectTokensX1 = projectTokens.apply { put("mod.name", modName) put("mod.title", "Kafkatorio Events") put("mod.description", modDescription) @@ -82,7 +82,7 @@ val installEventsTsSchema by tasks.registering(Sync::class) { dependsOn(typescriptEventsSchema) // dependsOn(fixLink) - val outputDir = layout.projectDirectory.dir("src/main/typescript/generated/kafkatorio-schema") + val outputDir = layout.projectDirectory.dir("src/main/typescript/generated/") outputs.dir(outputDir) from( @@ -107,10 +107,10 @@ val installEventsTsSchema by tasks.registering(Sync::class) { val zipNameProvider = provider { distributionZipName } tasks.distZip { - val projectTokensXX = projectTokensX + val projectTokensXX22 = projectTokensX1 inputs.property("zipNameProvider", zipNameProvider) - inputs.property("projectTokens", projectTokensXX) + inputs.property("projectTokens", projectTokensXX22) archiveFileName.set(zipNameProvider) } @@ -131,6 +131,7 @@ distributions { main { distributionBaseName.set(modName) + val tokens333: MutableMap = projectTokensX1.get() contents { from(layout.projectDirectory.dir("src/main/resources/mod-data")) { @@ -139,8 +140,7 @@ distributions { from(licenseFile) from(typescriptToLua.map { it.outputDirectory }) filesNotMatching("**/*.png") { - val tokens: MutableMap = projectTokensX.get() - filter("tokens" to tokens) + filter("tokens" to tokens333) } includeEmptyDirs = false exclude { @@ -155,8 +155,8 @@ distributions { tasks.withType().configureEach { - val projectTokensXX = projectTokensX - inputs.property("projectTokens", projectTokensXX) + val projectTokensXX444 = projectTokensX1 + inputs.property("projectTokens", projectTokensXX444) }