From bedfc1dab4d137ac9388d812c45d78e3dd985a79 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 14:26:40 +0000 Subject: [PATCH 01/93] osmosis monday --- etl/epr_etl_example.py | 2 +- etl/fileReader/pdfReaderToText.py | 1 - etl/fileReader/sitenotes.py | 5 +- etl/hubSpotClient/types.py | 4 +- etl/osmosis_data/asset_list.xlsx | Bin 27461 -> 20673 bytes ...osmosis_monday_to_sharepoint_automation.py | 86 +++++++++++------- 6 files changed, 60 insertions(+), 38 deletions(-) diff --git a/etl/epr_etl_example.py b/etl/epr_etl_example.py index 866c75a..e0a7032 100644 --- a/etl/epr_etl_example.py +++ b/etl/epr_etl_example.py @@ -1,7 +1,7 @@ from etl.surveyedData.surveryedData import surveyedDataProcessor files = [ - "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", + # "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", "/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf" ] diff --git a/etl/fileReader/pdfReaderToText.py b/etl/fileReader/pdfReaderToText.py index 9668eed..c1d5834 100644 --- a/etl/fileReader/pdfReaderToText.py +++ b/etl/fileReader/pdfReaderToText.py @@ -33,7 +33,6 @@ class pdfReaderToText(): self.all_text += text self.text_list = self.all_text.split('\n') - pprint(self.text_list) def get_list_of_text(self): return self.text_list diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 1df8747..7799252 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -298,7 +298,6 @@ class WarmHomesConditionReport(SiteNotesExtractor): # Gable 2 data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding") - pprint(self.raw_data) state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False if state is False: gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state) @@ -1601,9 +1600,9 @@ class EnergyPerformanceReportWithData(SiteNotesExtractor): class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor): def __init__(self, data_list): - super().__init__(data_list) + self.raw_data = data_list self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION - self.master_obj = self.setup() + self.setup() def setup(self): pass \ No newline at end of file diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index e004243..c042d6d 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -81,7 +81,7 @@ class SubmissionInfoFromDeal(BaseModel): missing_items.append("Condition Report") if sdp.epr_summary_information is None: - missing_items.append("EPR Energy report with data is missing") + missing_items.append("EPR Energy report with data") if sdp.rd_sap_xml is None: missing_items.append("RDSAP XML") @@ -90,7 +90,7 @@ class SubmissionInfoFromDeal(BaseModel): missing_items.append("LIG SAP XML") if sdp.epr_summary_information is None: - missing_items.append("EPR Summary information is missing") + missing_items.append("EPR Summary information") if missing_items: raise ValueError(f"Missing required items: {', '.join(missing_items)}") diff --git a/etl/osmosis_data/asset_list.xlsx b/etl/osmosis_data/asset_list.xlsx index ceada9c65be1198d05dd8027d96138f840418fd9..822b436c5566f8a0ca16a938bb22f974b4eeb705 100644 GIT binary patch delta 5098 zcmZu#1ymf%(q15FaCaxTyIb&Va1ZVt*x-T1E%+kAArL}vCwL$bUA)%a^41OPw`8B$?E z1};w-c>aZlP~lwpT7HdF^+&A?n&4iD1$rtAjrq!SkE2BYNX~7pov^}tHqMd->)QSH zKH=vmg{iJ}$UZt#DdlD-1}Md~S{W_YG?4V`TpQA`q<0M{s*Nbth0CXt=CxiyWFN88 z%F(+dU&iYq8JSy9k&$RX+jVr$H8>FCnB2u@EJ1`mfO&Hx@Q5QDc~ z*cW_D`>i1r-3JpZT9LfYGoRPi*^H0xN3?T#-~t zb_`j+F85qPJCXdn;Ugu-Cyp0ov4Cjm2Vol59*8@jT@3vo-PeoY)vo?hnL7^1V97>V&7^?hXm?kQRuGC^~wBE)T;AT>1hA_*EN`y%{?Y)YK>L%g*ZOBL#csi|k3rjjm^x5uM22nrA(r&i$b-y#+ z=_R?Yw!NrBwPmh{7`?BVwe#oWr&*l@H?KLLXZLH-3*cYyjVSdo<)4C$>IDu5b9{S* z(Z7`S9}I-Np(%kOKx7{g)?ia1fEjrmDKg=IfZYwtr8!jIg z=h$SR`YI1DJTSj$=ohv7DxmL51o6v7c0NM+fkZ;`%&a&Ic zVdi(Ub#E2~9-`AZ+NFm9*|dUEpF6agCihJOePv^!y4glW9;=PXGNyyC`)1{?j9LKa zUA|}~bi~otR-8AwVOT|8*Xs`r_oP8ck zcv>sZCQ5(vM1U-xF&qWcAT|X^dNW4249d2$)_11_pGe5 z4A>6NPa_hYX8}r@jykwD->JuGC`**0i(H@fw4SJ$36_+_-@#_^3oZ(Qwm15F(!@*J z=crv}<)mNgbN24|4ZhxGRrZyk>{tn=N~4jU`*V;#bUCk?BJa1Z84yuvAL= zF+620hyVaN{O#k+ZRTO)4C3~7^Ki0sb91^+AJ@I4Tnnhy!-L!Y_%@QqJIjj*Z}t^8 z{jTl~4%$FH>--Zs{=kz(3bqy#gDEv`B)16P{h>aub3yDCB!bf}_L3O9~j ze2$>+R8XKHY_NVU=qgNmw<4szsfjj|)pq@^S$_TuB`?I6X7rd0T}1-R7clyBf{y*7 zOW_RdsO{Po=sWDxRd|KDPFbc#T`DN~I^q>=WWN#(m^B%+PWmpRRd;{+mE3PAYv=qC z$Kc7n?b<-!xEiV9K_=p!gZ`9U<$AKE{LWD3j#WIIu&ca6D ztw2l0jdrw8mYlq>u0=R0OYX|=2J@dd+F(JR{nYSY#$nHc@64F@ni8XD5cSrrsubrsmh%kV!=f6-vw@Z7m9q zpb0H7qi^Og!91Cw<2>JnkUR4EWxupFB#gZcqaU!7iSAxzUk1!}j7{TxXH@=Vu*ZAp z%S%8=6|hcqWIT{iFIEb3T#9lda1LwS!n@yx^-DbVs8#gwu2_2}Ld}WJ%B-%Cns#=N z1l0##jo(xsTB%uYY8T0RoJ3-h=<)3%8*65MV;?n2xEP4F5G%!4OM_) z5%dJFG_L3L_7irzZLZ>v&7SH$tn3jFFprr0c^FoNa-(il`X2Q;#?RfZ7AZOIi1dqw z(xcYd#>{xOD8=y;SFx7)=T%C&QijgQWsW)O8*UwO&ZZg~3+bIzs10?OG=6H&;~HDs z!5yynt)$mTCf4JOKdF5DfWPe?Erx^Nr|VU~^*Ej_OjIDEw*bHQ(B`!h}*+Ys}+n3f9-xi%ix>NeV>A zCCQ!WNE%V8{01xI$Zo5PmoWy}T6toh7> zGD2&PopXnZEl^)dQ0~Wi=DkhpM>XgDj9dQfOeWwdzcT`m3n1syL8bGu`d5U=O2UOn z=fXbE=AB}hcij&Zes1u$>q$v*cmZ8O1psjVT8aN$gXOyOuB$@00o>ogxEGaC6GOGQ zb_ML@v@kA8VTU~8*r&Zu88!1uX1Xlz+T{m4@QYYt>qCGs*~=OHB=KCGl$morwzZq@ zOz1p8i5<-xXYEyDg$rUJ-FA#Lvhtn0QEN*dnwy; zgWGSYob17LVpAbS__YNXX-A{m;sf@x`=r`Hms!%^dh)(+o`h-ShSHVBkZ=(7ey`R8 z(k4&pAlwA@wFW9W#ekN`+43TWIi8Lq^dMC=`;o~ejS?=OTCp<*lyvPO9ma-+<$-a) zAr{qkYP478rOV-j?Ijb)GR6d3IJX4Oct>21A`^H;a0VI}{WruSnf1o*7}@W_);mM+ zuJcCGQl<>p=^pJv>|xPaag|)`rgg0!Ut!I8nmH8m>oOD;w?kDc4VvX18^lQZH_=6M zQTt$N`Kv^QmX~RZf+30nY+@6RkHO&*U%m_-+`Qj&7z)MQGzV*s1)cAlpkTtr?0EKR z3nlb61(yhp7EFC|3AmSia%UNpqjr3PH@h`^heY^W;G73IFkFN{3 z+wL?akMxJ7S$4`L)}{;Z|91bKo9-wKEk1RNmHrj7Uz_)ooMAZbJyn9^( ztc;(}IcH}cTkZlBgqfSSl9_~7XS{*jLsV|EV~oc~`n6)Tlgw52=5dA}$aF))+>jHg zrwgc$SARLwWeUZA44mHS28zKNw@x})VIk*&;@YB};@Z5IEwIXwiS~iU#r5n;>Jp*f_K@Ea&RJZaBbVI&O(#`0Yez|v!)y}0ANZB2^6CR54g_X z??;fYupjLcGx<7L8eq?%D1sa{n}Ah>g0*NMO%>?lm^S~&BjV4II3k2#alP3gHjTQ-Xnn=o}u>zsNE9_-PN9)$Pz}Q4m`^M z_k16jyN;v^>CyU6$RC=Gz4v4c7U#@}n?lL(~pxDy=`_ck;x7P&3|sL2re)lLr_7>u=umX&RB z=-uqVTo!T*Oqd32a$s-Si0rk0zbsESSbj_7u0e8QC3GV|ttFIqiglvaJ=N*Q9GtGE zL5pafh?&(`Ghbz*G4|P`O0BF~p{9BD2*iNIVa!!s`H4`xx!#K^NuA|6@27oDUJ(*m z*Cl?IaB4-g3sCK`|DgytCU9o<`_4I}Spf{_5Y>_$5+~}I-y=yZdy+rc%0gmaK@**J zh-SHYXd>-qLzEd3_?FR1RrSDvt}q6F^wRwF*g{-f4%>snn7i+iWh6r7)=YB8XEFhee=oIs@vU~$!0>uMU-x8e%e|j@yWUD_XA1~ZKcY4 zc`jT}L^bzQvPT=IuMm^xGK=qqD)g{%tv6*Nyl^f2UID)*enNo>dVNpGs1rAX6GaG;OC%V`r9@Uue>E*$tlp5>i zWD`IL|NKOUSP>LK7ki58e&$r9Qp$4idvDlKbwrK4U)6n!Htrlry6<)4TtTsX*&}sC z^PzZw-}Ng8j79HN3=^;?1*&6FGzK5v8?fPHH zN13e$D%hytTnabbob;6khO5q$Um#}@=@btI-hyCYKpln5H7W*8#<^)YoRYbrLgY+S@omm&s0`21Y>*IhoT-&l%jn8oWobs`g$DSJzpD)fgQ$Umw0`_k@uI&5M zrE_FZVLN&Qh^UWg7`WJVn>_TF&YSwO2pL@7mS<=rlU(!*nx;+bH&3A$mQNtN#?CJK zd zW%**Gg%hORQ5(OR%L>|mB{e;zO5t?$j{3vTj10w>UQ75cKsRLWYHmD?DVkLe<;?|r2?+z*V9hFDb0-6%=u$Bx8x&P3PSaEal^$9!muR@7^GFzYw3cVbsHZP^k+^1D?0cd7PFDIKqWf|GK& z=rwU|Ok|4~gXYnH#9O$C;5uht@CZBq;3AI!qW_3!V4cI(qITk9POB&5X3bJG_yBR1 zc>`dP;S8Dxdcl}6^`_W(&5S+w#v+aX&#?glA_m}po#y}m1n|Q3j|)<5K`f=A_TTuw z4;$cg?eRar0Ki9hvj1Xk4{Ys05i)Ac#QGHu{hN?mynK02=>Bc?fxts(Y=~r6;U)BM z^nd94zv2HTjei^|M}~wu&>~1o?vj>9LX>Wi?ve)SM!K7K;19jNf4q0^ z81L;d_8DiNv(~rfoNMiG%~;Ku5bfg-LkcBknG_Q_XE?m1UxFRqjRHOy z+D5LVcfX|z-BQjXU;%=x_aWV|mn zx3$;m7OH$0)(!$n`T0WZ8+ZnQL#?6h@e8<8d@P7>EI9daAOi? zSLHCr`0%WltmoWEw{Iz7rkV6>mBekHEmk8&JnlVX!1QiAkg~Xq?z4-DT zxTfbA&tsd6=ZoTCu0j4ONHHHl**;4MQo%g65K{Xnr_4k zQz?x5c!Ya<-D?_oMAx~?g|;;UWmrG>X!~U&K2+(k!qR~ytCW0d8z+7~gUl66{tqU( zOT#ACpa~mH5+Twd6wBx=083*h!FOafDW@q*;}vqgMeb&(L66WJGDe%vQq^i&dl%-H zMAt&5u+;IEgOWCLPo01<6MKF+zuWw$g}HE22pii|b@u*RNOTA0imRaq`Yd<~Xvl9i z^_W#SBX(gGxl_5c`!rRF@IPUU1|)Y083oS7NN@*F3m;#oQ_10J)aPQzDlbzN^n zhdB0+lhfZrc;w&4YiU!d^z&E~c%ToR%?}AN`>tPGz`$qYB# zKlr6U_G}Egp-vkRTicAR8gyNRet6nkAALig*|XO^DLum}Lu z_h|24eqL9YHTBPF`AR;RogH5-iDC95zP2uC(Od6Z64B$iv!zNj_cc$tvUYc5e}-b) zjdbp)la>%)f%=T21FuBxp23~9S?x`(3?gp$jzZ0*;w<#+K=`_3e%Cqi2*|$RDDOsK zq61j%Ng7^*8DbF*1VRNffr|y5v$dVMp0&04Jx3HPT3Tnbq1}3YyMb$S4w0mX7_IOH z)rs}bQTJE5zo&j8p^rt+m?v5}ednbQZ(3BMB)cc0r$2a}WUR%Rs=fQ8$Hk~@IZHM1 z94QIb+)j3X%~scI#vs8OFD*-wk7gM7{Pv>d_0i$dAkTm^KQ|ZrfdPnjgtqUaoTbT( z<`+YjodLmKyr?+-afN0%?rBXLT-rKX{lXFpIO4$)D(k+-#BupLL1ioK;(~J%#j+$p z23U%`O*4m8DMMS=m^H~g`ClsSWUE03UujCTU}Z)=9PMwFK91p5SU4rX(5G7j)|&}Q za8kR`gSZn5U0(`Oh&jRzor+xg-Qi5(G$A(M>LW=B*jaX`*A6U3#~*Wd_i*UBe$v6Y zF=j@}Wf>KNZ@wbMW4E*K3q6Wa-gbJS=~uP$MD2%!VxH`+}gk2@ReB2p01T@Qahe`;fA^v*6-CZZl6G3 zh=8LO1b4^TKx$N<;|P2#TVf}hq%m|Yj=OkVKmIErshAiOlryA4nm(AFvD1b40L{dg zHT!%OSy!UIvXe(k@hy?6tOBGqP@yhf2DM&ss?pL>Z_Aid$Z!vsR_;bifQ>8n-l-Dz z6K%Ti8+>boQuV~BvUjic>UFOoo|tiTb)0C2@mTrf4}5Z9e)skQ3ZDRRO%S!KX8|`r zl-1_*d=!owR1~gJHtwNo_7zM3dCodbf-vG6{JtGOi^Qchryyyek*xUDZXGvQ)mR&6Tqx?SG%4`biac1SR#E z0Hnu!On5M1Hm-~VQ>yAqlqjsubn|G^lxTLT1gdVW?k(@Q_YIj}xKdS&6i(2&v3T87 z0^Ryh>O1@GzOxxzY8D|*blXgIj5NtNKtW!X8oXDuD}nw7tnJ$pU-ecmviTDXWU zSZX42{Kv+7$xBa0-8;sN(4DRwN*^ZrrUA z4$`|%n+sLxq%liMjd<}tVs42mQ(Zqk`($V>7JSq|Rm+IHatw;#?&krLcfw|C@V;V@ zGQZ5PJhSXb#9%Smz07+H>rAcBa&``wPV`k`yWk3`mGZ?%B(9ZZ3iBK7=_VG&Kz_j+ zHZ{3^tjA{?d8^O{G#pN7I(d!A`f2oJd?Eh}9Ju~mo1H9%IHP%`Z|zEOCF!PtxJgOvrlPUt z1fv&h^tJHKNq!Vhua2_DZjVnd-EL1kytsFH=I$)HQ3P;U^LYlFq<8Y^Y$qGiPzAov zzMZ`;GSs?FAJ}X9(sX;xK;B~Wy(w%&yYrwmDnQcxc$DCVbN6&+aM|lhFF2nn+Bn!o z)IvR2Joa>3EY`N+j{O}u@Hkl_`1B@5q&lU7N7$4xf-c3tgpur-4g6Ayo&4jG8CC3BiQ^bmgNPf$s|d|oEX zE0Y(LEjDSYdC3`9J3V&eUKW-gPQ3$ddgZl9lVPI^^ten1O~bCSYOqvR+WI{ew7ufI z&L7g~?VBLiVA1eEhl!r$Ms)wNv!=zc6n&QpoPy|UpLM?}7PKN27Se z2OXb-bzGEy{qxtwMmxTETBc9(N^=`R3bli+TSI)0iyG8~6>9gjYKg$Yegxa6);2xx zK_;#d{dh!1m2(yAllK5wp@zz82>AC(u)1#*Oa4?Bu}I@jb)P8JqWd){Yq{?z-0QeL z0&c-sBrgE3=a-h+M;BvWa!qfyvSfmOxp@G5Y zY*>z`8~B5}E?iS=O_$rOORc*eGxa`UgCtbN>Ze8HjmRr{UiJy1ivr6^{dvw#L`}uO zJLUFG(OplpHb*a_31SPn_`}}H9Xw#p`Xk9o<-DV0#_F!*#m)dLO;Z)C!_(YsN=40) z*Hj5Nb1;QP1$}dsd+r?9;W1E`P$1 z5;qZ894#g2!Ni5siC5;}uNx3fwuwm&NYl^gmB@C5(7g@AcZk zsb`20Hj>23Z;jQaM_)?6FL*Q+w%?RUd*PH^gAiRxpe11;!n6k6fY$?|)szTh2}$j{ zWB_U;%lXoe837DD5i#OqzPRn+Ao+jRtl?`_Y@0iHPc=q@NCLkpth=WzkBT@p`_Wa_m2lhIo~+r;SE zb$rF5{CJxV6Ea8A{_1|G1seVJNTR?KI0L>hz_-4my@R!-ilqgese_>yp*0FKWm&Mn=Dz=^cM@!6O#zSv<}Glnr9 zXYN$tyZ+EM%0MGINq3NKECmfM%sW+=(QyVJ3D&DNVrot&+Wkt?%4QYKZ#dh23fc@@ z^&DfP>>B0$hN8sRLSLR4``vXlNzXV(X0V z_Ej}Q6G^{{w!(h7E~P}IXE$0O{D>Bcn=Yenn4h30L}&Usr45(Wu%vo$`sT zqLLcxTL!z?oA@%TiiFNL)p=TX*lASUL}kf`k`*@6o281>mmIK9-d*$d*etNu#QpG) z=S0fxOt@hpB2zSJl)X(lS;Qlkxq5JYfDGFf!Qn?B5IWO8O$g)tglwz-u%2NO;Lq>gmf#BXx_0(3`3ZO6qYTY|UV?XD@bvCgOtX>!_9Q#B3Uf%%#P+ zD1k@k{YY1f`8TX`>7p?K%vyzO@tM@59iynrow-1ADX;z5dv33<7CxqZB7NhZI1asE zfl~=C=j(Ec@wC*yxYi|%Ww%^oF53)bl1tI)&g_mH%W1$P0C}WBDUYe%mU~cAf2sKU z)baF}T2>xu>tgfPVlg7PzZl&YM>YyQ_DTC7Os!cgr%oLdU;DBK zss_N;c>uXxgRwN-#qw;vDzEK`+-DQr*sMRg(0ggg!DmWtmcB}T!;N}0M$1wlbtn#Y z8)4+5lpj%>U^S9{q4gsHZatAtrdc`_Li#v6=uzCd#G)`q<{dFZy*Q#{kvzBiy6-O$ zsp4B?4n=Ty;v53$EnI4b#?<&#ALV{_PA~ujJ&^d(FQ}C9)7_3l7ZTqr3|KnrEm4(v z`l5YsSaSEuptdy=jSZGpzkMG6=rF;zv7teuaZ;Jtb)}B!glD%&q@D(|O8Tm)zTTlp zjyuY|;5oo59KbU2d9j~)GB(qdo1r3!L#V@JEqSev+Q9#YR&nrIW!0Fej*=%(u9+S7r?PdFX&fI3UEck6ruHR=;_>1wHMB}$B?%oiC}mE_ zU~rk&RBy!{mM{DI`--f~99PJ2wvLw7DToeJ2xi7tlHWJCM{+IwoY;-1n~81Ib5~5q zUH2Mm`2~bM{pbKU3OewHsy?V0itiCH=RuQZ9fw(smNvc-+?M^J$?c9baTXo+?i*=j z@Ft}aK*1o2hMN4trnk|x<_!;*-|#h5ZVJ{3oZUJ9l5kaRC?UL{P}3dC^Jjf|KI}<4 zOB`P!x!@WGE7FKQms+Z$8M*DWY84NAy~UZ+5FW0HRk?W`vWwWxznL9m-9qULJoD@c zaU{642hf|Uze{rVriERlo~Mp`)bd1kd8{)^HGNG&JWe8?QAf{s;Y)hNB_Ip`sN<8~ z9Y862K0{rUI=DVNl*bels?*A-y=~LffYAXlOI)>>WR(X9Jc0_;fLIVSQ7}BTGTBRO zjYM#JZhfrlrM6EZx+;Pr97=x*bbA%lfzZCzML&U%pgMRrJBsR!TJ0|<6RUc5%yZM+<^=Nx^MRjugtKn?<8p-_{L9} zb`$zBRp+H>Ud5|H;BX4k;Il$8k7RoCOtVM1h$AP-)Fa6b^JzhiIz-*x03Qlb=&13N zS{+6W?0DH1N}=^^Q33O^IvHv6GGpr%8(|-XT;{Jw0!TlgJ42aWeD|(ojo~De%qjv4 z4nX-3bFSW(H5ozwdqY{gRej+tOkZQ z)v9y67!+o1<>@fKK}-|5t7iT|i_~&_T8B8tN3EF~@nTtc2UH<`m0mv6y5-co0SELN zl<^mcaLM{LDk=J$6G(`43>btY)(Alm1~C3Lo^*ko?#8Z zEQxOcHk~FVg0kajvQ5la0StXC>om_0ka~dj1)$R5i#$gzXENiH*_|6fo*%TsHQ`SL{crubs2G6ua z^E}sCvOFRj@zxrm3<0q8OURj_8Kmq14XVeA_N$w_G!;hEd7o8gHI2rOLDe{Iy0w(R zE?#we&)y2jGs=2G;?1K{ChT7vR7yI@wCWy;xjn0d1mUN!f?~ zMdmVYr!aV`+qPqmQ~U$M-ck3Qw3McFwd(dTQnD)N-h}(6r3oFLLEub@%)D51Z}G_N z^wTqAbz~kCWgsIQmzM;)Q~_y8$L<`O=o3?)M3538O8O(-4>)qHZVjp$KMvgN1T?STAtG;Jn1p-8QN|>v>b(V`_`=dgGXIEPI5=> zxH~_0_FX8*!J=?;8D^~-)8tzOCi1?QOW`Jz1+04Xv%1o}CxJrnJ9Zt$G*DyVnFpj{Cl|~8e=<1hmMX&lNQg@Yo zEbUPnB9z~K-rpAFINJTvtiFK{6`5;J_m8g_ z+>AHfdy@^w<^G(3oS#rvlZjaTIQQ}mH0IpT7TNJRv5L9ft6yvBZ1mf6c z+(>0jGIBO>(}xAA{3Slcu{8KN)dykNut7>>eElv9qM+ZABbuO|La~DP2?SN!J;L

Sg0rqUUepQhtfWdVI2bH z<qx^YLb_-bXkkw_aKM2o*QYk3EhfeOdDhRs=Qrx0-MnbXyaM zr>5+%w3jN?V^P(Hcv*mA!@OEu#+MEpoma;P9U~$BBhWi(@y%X%R}1g!m1!cns9W9j z-=mw_t73|?BUn?i4lPrDu%o~^Fv%rU6}pv~b`8InlkP{f#pDoOl zjK%Zgpv`7-Ie4+66Qk8Ok|N-UKYeZ6NP;^GS?&y93XdgBWPb7F5j<({ z4lgQlwK+*Okq7-~a`Yn>wAKUq_gEk2tI2T)U4RyGUWz69wgv^fFPmi&M|FJ$TCd{! zVrJD6@*CnD8)x!V$8y&DH{qTaQq8#_d%_Pk%FUf_glX?%P_;x*81om82-r>_W7Aa8 z1!9g;i`p4#%kJ%3K|7vO&EC?Jwf`~y5JnU{Q;BS=Rkdgod51|kI z9foMz8Cux?3`SnwKN4j@Lx2l#;Ku)77x4cD{BPp^yW{&0HitkmAi*xwFL1Z|o zZwwnVG`um5D$9q0al3vsi)#_E$bE6Jf>D$u$VwlalQzChj+8(nhSO`1@vJzGG4i#l zb0x`^qo%dPttIc7ZlQoR5#c55=tYv);Tg2-jh;`;Coxm*OKdl6f1d8XaAQ13#Dg47qcv$)WuPfo-wg104o&RyO|9pJ# zN-%4s2d@P8%sm|k?bK6xvEUJL+!fY0h>du{q7wSODq^PdUU~W0upDCi^K>kM?ulc_ zOzt7Dtcu78+1YcQ!(T6sPfDTM_4KWctwT%}e2@a28qK_Hs^k(}4uchCpFd3${HA5@ zgaO-&MHXhrD1)LTmL!~3TtEHzq^g9+(TgR+lSy&nX_sabgri$d%ypAw>jFMOH!Vc= z_2bOh7XC%_wLOX8P>Z7=y|0J?#^&nr?tInZURR%PVD32&VLr#05)b@R8$idl!4Lrr zagyd>jRN-kZs3-$vPfs^7`KiYVH>S*x6Pid`gJ&c9<#hSW5N}er<@`?){F9=KP&Zb znDzXT;F)z4&f-fU+lPy17}}xW+q6#S{B(d*RK9B*53@md(0OX7{}fAIbTQ^r%0PLB zUc&;5>>2ceVz}C6my-#9ZiSRMLMBj@5xYf1%knG4)np;|T#|Loda>dRQ&s zgvMbhhmbT+GV~y-3`E3Y2S3H;MP~AHI4pm4cM>fvsY57$wKaA^i z1Cym>*l8od>A(tt5ZS*!`E#bG8*>LxQ{fNI-qI()_;gd#<~8}Z{x(xf(x|X+A$@4I zOVP1(2zB}|n>~C7*6r;EFxDd{RK&DSBL`oxva$+Vk$L1`UsP{)m{sqyX}N}R+7i;9 zrG2o|^Wt4cTx#)}Kol;>0Bq}4v zR}3h2geuY|jwcrg55EURXQt#B#Y^?d`BKNlWMv?WAMyXtSaZ$e+1Dry7zxuAg=u|D zNrjC-Y2?^GA()o;LfsH`Vg2gl&e+|K1O@*>(!ug#Q*Px-9myK}vi|oS%14o{?XRp8 zfCHlZEm8K6c6u;8Q5Grqa0t==`o{m3o~o#-3X!$ZYi?|@_$xhy6!dp`>N-%g!6t$v z)<>^^kH{6+6|Tdr(Rw}Q8`LekM^4VnixhaXWIWMnxfc@}Y|Kf4^c+=xpqypjDn-n> zQ=4YuZHb|xQ!eLtzvy^-hgDBu!FJ(|cb8eutKqL)xt&S{8+$k!8i!C+tv>4uawSKS z2lQbCv{%NwVuhn_AFr8yv}z~v?lk>v7vR^e+x-YK8%)^$BVRul=Kqeb|8sKuUuWY# zYVdzD{!hNvec)?{d%o_a1oJgqJ)W>_pI)!ZmdYP|&2`V$CxP=dg@iAjQQ~d@$=I0W z!tx?>U)ML!8^DA;VxTN$vfzspxM(^8ESTlKJV0kGDkNq79JJCnzKtE0@U$m!%ODT^ z<7k;n+Q+LZf|DvsZO2*`of@VwHeAZ(QJ>eEc9LEeBb;!7)++;#W3MqXXOrG8DQnCl z8kUsN614N8g-wi$%kRN(9bfK3H7%mv6ZRtvtuKa=%}{R)8vh(j@e3$$QAw6Bq4 zhWhz83p$|q{ZjCMd6R46Q*$L)y&Bg?vUc8EdQ5eciuRT!bR9xz&xHzKXdzl2t0>9R2KJn!$kw@E6Go$y>8asxg<1Ed4dB` zs^?Kt{98vXSxT9?q|KkbdNzZ4uAb{d5m2bzyx-U}AC{AQL-sZqDsAmzn+>L-AxFp_ z1Nffw#$%nnfe|%d#fq*w>x`k4OMgIl@D=*NT-oIxFGlP*EGj^)VW(LLwuk`uRLFfg zU0uN1%E8deL0iGq#?W5#o|!A+=cQ-p(K_cTc7UMk*DNn3_)W}uT45u-p+($T4k}Bx zv=22iZ}yy>AoV_&cYbSZ3r@T8;`wq!RvsE&J_asOH@{lSNe@Brcv^AL`-bzow&Z}& z+nfd^Ru>bX4O>Q|4_wTiwI=V5uj6NX{mYketmTy2pW6-?FXEXrdS7a zE3p%MHRMq8{n4xV+Y)VbNVjl-4Tl^f>VCw#0kKIySiqJNS)Ecu_i9JadX!Qmaq-s8 za}LT@Jzce-Oihjf)o=!%Z<7#F<&L=K`C-7~-0EO+HI?8>8^t+Wvv|7QF=-hZDdT=} zL1RQ}=CFEz#|!3-NVjV9oP{?tXN-U;cU;{lj3KU6gvOCaT*&^H7$!JdCVl`vQlaH2l)_%`K**}(152C{m+f|=TX21uk?O8bD|%- z6y66f#ocVmaypNc>cS)=uvuqzm)!OnuO-|}zgoxbl}1&`_UB$F%$W4_92d_RIA#QW z_mAij6yDJ?l~!(JWaqBRFydjw%Y}<@9|e5}Tb3N%0N@D^9vA$wn{Ko)Wy(UMeQOAs z`rg|Wr=^h|V>L@sXKrTvVau6`tkDx$gjIN`#Mn!dw1npU@YeSYV;i?277m`J<#hE& z3$CWuFg)2xbzemH)p4;6artpRODaHPnh*$%m^}W9?!8+4AAw1=u#vhc?3b_|1gu5XH52N-o^C9Jp@YSnH= ztlduB(If}o`{Um5RmC{#b#h#_9#vngoc1069rZy#(#HBol0krC7a=jQWW>SG4+X?O z;=qTam$8GASWof6R{v@3KM!91(L8nr77ytsVQ)dkT1rt-{tD9rJ^*M0p0Z)+KT1SD z8w*3n%E05nJOIVcN|8NVhJ84?Woc+&s!L~TscUS=@Y2T0_>Tqz(qvp6j1LX$9b7F8 zAIh4)%l!L6u%ClLu*P~zV}TWnRpa_qViWe*M(KZ*_~&7l-z7GB9c#ml2m7b$Ljrie z>^>p953)I!7+Mtr7zFy!P{&7yZYg;qM5) zSM7cRWQhF^@Uw>ZJIZ}=4EVJ=_7f&T{CAj#DDiiM-}5Rz0iqOt2Y8U*^9qK`YZYIXUO}f2lXqKpUOXAv4iWfKLg;uz(9XyQhvrzAQ0ZK zSfCKK*e;l7!0#P?k0pNg`L6!!==bH|2S4sFfWO55)ib`Y$Nsz|HGc-&dss9n? zU;W6RAYcfsUtxY)^ru((mq>pZ;cveL%mLsm!!MWFYaI~i4~PeU^!J{>@83VW#_Ptu pl*E<)m1ch59ey^p(f^gb{@yEO#9_d6bN`0pfb_xN`V8)${tIM{f6xE` diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 4e83612..40118f4 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -8,57 +8,81 @@ from etl.scraper.scraper import SharePointInstaller from etl.scraper.scraper import SharePointScraper import pandas as pd from tqdm import tqdm +import time -osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) +osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_3) -parent_folder = "/Osmosis ACD/Osmosis ACD Projects/WCHG/WCHG Walkups/Property Folders" +parent_folder = "/Osmosis-ACD Projects/Cardo/Cardo (Wales & West)/2506 Cardo Property Folders" -asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet1") +asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet 1") new_asset_list = [] # Create asset list and location for index, address in tqdm(asset_list.iterrows()): - folder_name = address['Name'] + " " + address['Postcode'] - webUrl = osmosis.create_dir(folder_name, parent_folder) - - first_folder = "1. Retrofit Assessment" - osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}") - osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}") + if index > 39: + folder_name = address['Name'] + " " + address['Postcode'] + webUrl = osmosis.create_dir(folder_name, parent_folder) + time.sleep(1) + print(f"building folders insidea {folder_name}") - second_folder = "2. RC Mid-Term Plan" - osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}") + print("building retrofit assessment") + first_folder = "1. Retrofit Assessment" + osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}") + osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}") - third_folder = "3. Retrofit Design" - osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}") + print("building RC MID Term plan") + second_folder = "2. RC Mid-Term Plan" + osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}") - fourth_folder = "4. Post EPC" - osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}") + print("building Retrofit Design") + third_folder = "3. Retrofit Design" + osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}") - fifth_folder = "5. Trustmark Lodgement" - osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}") + print("building post epc") + fourth_folder = "4. Post EPC" + osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}") - osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}") - osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}") - - asset_data = { - "Name": address['Name'], - "Postcode": address['Postcode'], - "Sharepoint": webUrl, - } + print("Building Trust mark Lodgement") + fifth_folder = "5. Trustmark Lodgement" + osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}") - new_asset_list.append(asset_data) + osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}") + osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}") + + asset_data = { + "Name": address['Name'], + "Postcode": address['Postcode'], + "Sharepoint": webUrl, + } + print(asset_data) + + new_asset_list.append(asset_data) -# Osmosist File strucutre +# Run this is you just want to get url +def just_url(asset_list): + new_asset_list = [] + for index, address in tqdm(asset_list.iterrows()): + folder_name = address['Name'] + " " + address['Postcode'] + webUrl = osmosis.create_dir(folder_name, parent_folder) + asset_data = { + "Name": address['Name'], + "Postcode": address['Postcode'], + "Sharepoint": webUrl, + } + print(asset_data) + new_asset_list.append(asset_data) + return new_asset_list +new_asset_list = just_url(asset_list=asset_list) df = pd.DataFrame(new_asset_list) df.to_csv("output.csv", index=False) From 982dd3c5982b6c5e5dfb28a9f732a940607def0a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 14:55:16 +0000 Subject: [PATCH 02/93] example for khalim --- etl/read_stuff_from_s3_example.py | 40 +++++++++++++++++ poetry.lock | 72 ++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 etl/read_stuff_from_s3_example.py diff --git a/etl/read_stuff_from_s3_example.py b/etl/read_stuff_from_s3_example.py new file mode 100644 index 0000000..cb427ba --- /dev/null +++ b/etl/read_stuff_from_s3_example.py @@ -0,0 +1,40 @@ +import boto3 +import os + +def split_s3_url(s3_url): + if not s3_url.startswith("s3://"): + raise ValueError("Invalid S3 URL. Must start with 's3://'") + + path = s3_url[5:] + parts = path.split('/', 1) + + if len(parts) != 2: + raise ValueError("S3 URL must include a key after the bucket name") + return parts[0], parts[1] + +def create_temp_file(content_bytes, relative_path): + # Save under /tmp/s3/ + full_path = os.path.join("/tmp/s3", relative_path) + + # Make sure the directory exists + os.makedirs(os.path.dirname(full_path), exist_ok=True) + + # Write content to file + with open(full_path, 'wb') as temp_file: + temp_file.write(content_bytes) + + print(f"Temporary file created at: {full_path}") + return full_path + +def download_data_from_s3(s3_uri): + s3 = boto3.resource('s3') + bucket_name, file_name = split_s3_url(s3_uri) + + obj = s3.Object(bucket_name, file_name) + data = obj.get()['Body'].read() + + # Save using full S3 key as relative path + return create_temp_file(data, file_name) + +# Example usage +download_data_from_s3("s3://retrofit-energy-assessments-dev/JAFFERSONS ENERGY CONSULTANTS/VDE001/12103116/docs & plans/77 Perryn Road, W3 7LT EPR.pdf") diff --git a/poetry.lock b/poetry.lock index 62d14e9..0074416 100644 --- a/poetry.lock +++ b/poetry.lock @@ -84,6 +84,46 @@ charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +name = "boto3" +version = "1.39.6" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "boto3-1.39.6-py3-none-any.whl", hash = "sha256:db965dc9019df7b1d20e8d8ab7a653956f275865175a8652419ebfd03de03d83"}, + {file = "boto3-1.39.6.tar.gz", hash = "sha256:e75bfcd444e199767642f28ef8dc4f972846dc3118e48a7e09f9c458dae2021e"}, +] + +[package.dependencies] +botocore = ">=1.39.6,<1.40.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.13.0,<0.14.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.39.6" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "botocore-1.39.6-py3-none-any.whl", hash = "sha256:9c002724e9b97cec610dbbb3bb019b3248ff6bf58407835621f0461e740af90b"}, + {file = "botocore-1.39.6.tar.gz", hash = "sha256:d3a6c207d233ddee3289c1d56646047bef18b21a1faebb3d83a6fca149fd0f59"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.23.8)"] + [[package]] name = "certifi" version = "2025.4.26" @@ -717,6 +757,18 @@ docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alab qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] testing = ["Django", "attrs", "colorama", "docopt", "pytest (<9.0.0)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "jupyter-client" version = "8.6.3" @@ -1682,6 +1734,24 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "s3transfer" +version = "0.13.0" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be"}, + {file = "s3transfer-0.13.0.tar.gz", hash = "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + [[package]] name = "six" version = "1.17.0" @@ -1969,4 +2039,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "1d5c1e0bfc12e88ca9b4c46141c848064a45e9cc4b60990fa3ec7ecb5ef71209" +content-hash = "7f71e898554f0285d708a349589cc96fe8cfc210e7d39caf66c1dc6faae010da" diff --git a/pyproject.toml b/pyproject.toml index 56b1bb8..84c15cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "beautifulsoup4 (>=4.13.4,<5.0.0)", "tqdm (>=4.67.1,<5.0.0)", "hubspot-api-client (>=12.0.0,<13.0.0)", + "boto3 (>=1.39.6,<2.0.0)", ] [tool.poetry] From 060a3c58a358e76c1829f0aed644de2a42e23414 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 15:01:35 +0000 Subject: [PATCH 03/93] update docker file --- deployment/lambda_example/app.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/deployment/lambda_example/app.py b/deployment/lambda_example/app.py index 05e2693..a520464 100644 --- a/deployment/lambda_example/app.py +++ b/deployment/lambda_example/app.py @@ -3,9 +3,24 @@ A quick example of lambda working a function in python """ def handler(event, context): - print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!") - return { - 'statusCode': 200, - 'body': 'Hello World' - } + try: + s3_uri = event.get("file_location") + if not s3_uri: + return { + "statusCode": 400, + "body": "Missing 'file_location' in event" + } + print(f"s3 uri is {s3_uri}") + + return { + "statusCode": 200, + "body": f"s3 uri {s3_uri}" + } + + except Exception as e: + print(f"❌ Error: {e}") + return { + "statusCode": 500, + "body": str(e) + } \ No newline at end of file From d1897024fe2affac75c7f3e8c8217a432c820ff7 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 15:05:07 +0000 Subject: [PATCH 04/93] push this docker image@ --- .github/workflows/push_docker_image_to_ecr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 7bc8040..6b016e2 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -2,7 +2,7 @@ name: Build and Push Docker Image to ECR lambda example on: push: - branches: [feature/energy_report_etl, main] + branches: [feature/elmhurst_epr, main] env: AWS_REGION: eu-west-2 From 39dd7727d3f32d4e7d4a1ce4715290e3e4756254 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 15:26:25 +0000 Subject: [PATCH 05/93] push images --- .../workflows/push_docker_image_to_ecr.yml | 44 ++++++- deployment/extractor_and_loader/Dockerfile | 8 ++ deployment/extractor_and_loader/app.py | 26 ++++ deployment/extractor_and_loader_lambda.tf | 85 ++++++++++++ deployment/lambda_example.tf | 121 ------------------ deployment/lambda_example_and_config.tf | 106 +++++++++++++++ 6 files changed, 267 insertions(+), 123 deletions(-) create mode 100644 deployment/extractor_and_loader/Dockerfile create mode 100644 deployment/extractor_and_loader/app.py create mode 100644 deployment/extractor_and_loader_lambda.tf delete mode 100644 deployment/lambda_example.tf create mode 100644 deployment/lambda_example_and_config.tf diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 6b016e2..178e646 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -6,11 +6,12 @@ on: env: AWS_REGION: eu-west-2 - ECR_REPOSITORY: lambda_example jobs: - build-and-push-to-elastic-container-registry: + build-and-push-to-ecr-for-lambda-example: runs-on: ubuntu-latest + env: + ECR_REPOSITORY: lambda_example permissions: id-token: write @@ -47,3 +48,42 @@ jobs: echo "Pushing Docker image to ECR..." docker push $IMAGE_URI + build-and-push-to-ecr-for-extractor-and-loader-example: + runs-on: ubuntu-latest + env: + ECR_REPOSITORY: extractor_and_loader + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # as of 14/07/2025 it'll be using user:Junte's keys + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push Docker image to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: latest + run: | + IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + echo "pwd" + pwd + ls -la + echo "Building Docker image..." + docker build -t $IMAGE_URI -f deployment/extractor_and_loader/Dockerfile . + + echo "Pushing Docker image to ECR..." + docker push $IMAGE_URI \ No newline at end of file diff --git a/deployment/extractor_and_loader/Dockerfile b/deployment/extractor_and_loader/Dockerfile new file mode 100644 index 0000000..c1caf03 --- /dev/null +++ b/deployment/extractor_and_loader/Dockerfile @@ -0,0 +1,8 @@ +# AWS Lambda python pacakge +FROM public.ecr.aws/lambda/python:3.11 + +# Copy function code +COPY deployment/lambda_example/app.py ./ + +# Set the CMD to your handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py new file mode 100644 index 0000000..a520464 --- /dev/null +++ b/deployment/extractor_and_loader/app.py @@ -0,0 +1,26 @@ +""" +A quick example of lambda working a function in python +""" + +def handler(event, context): + try: + s3_uri = event.get("file_location") + if not s3_uri: + return { + "statusCode": 400, + "body": "Missing 'file_location' in event" + } + + print(f"s3 uri is {s3_uri}") + + return { + "statusCode": 200, + "body": f"s3 uri {s3_uri}" + } + + except Exception as e: + print(f"❌ Error: {e}") + return { + "statusCode": 500, + "body": str(e) + } \ No newline at end of file diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/extractor_and_loader_lambda.tf new file mode 100644 index 0000000..1d74b3f --- /dev/null +++ b/deployment/extractor_and_loader_lambda.tf @@ -0,0 +1,85 @@ +# SQS queue for extractor_and_loader +resource "aws_sqs_queue" "extractor_and_loader_queue" { + name = "extractor-loader-queue" +} + +# ECR repo +resource "aws_ecr_repository" "extractor_and_loader" { + name = "extractor_and_loader" +} + +# IAM policy specific to this Lambda +resource "aws_iam_policy" "extractor_loader_policy" { + name = "extractor-loader-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.extractor_and_loader_queue.arn + }, + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = aws_ecr_repository.extractor_and_loader.arn + }, + { + Effect = "Allow", + Action = ["ecr:GetAuthorizationToken"], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.extractor_loader_policy.arn +} + +# Lambda function +resource "aws_lambda_function" "extractor_and_loader" { + function_name = "extractor-and-loader" + role = aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest" + timeout = 10 +} + +# SQS trigger +resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" { + event_source_arn = aws_sqs_queue.extractor_and_loader_queue.arn + function_name = aws_lambda_function.extractor_and_loader.arn + batch_size = 1 +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { + repository = aws_ecr_repository.extractor_and_loader.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +} diff --git a/deployment/lambda_example.tf b/deployment/lambda_example.tf deleted file mode 100644 index 0748af4..0000000 --- a/deployment/lambda_example.tf +++ /dev/null @@ -1,121 +0,0 @@ -# This is an example file to setup a lamda function with a sqs and cloudwatch. -# Please us this as a template for future lambda. -# Be sure to push the image you are using to ECR or it won't deploy properly - -# Create an SQS queue that will trigger the Lambda -resource "aws_sqs_queue" "my_queue" { - name = "my-lambda-queue" -} - -# Create an ECR repository to store the Docker image for the Lambda function -resource "aws_ecr_repository" "lambda_repo" { - name = "lambda_example" -} - -# IAM role that the Lambda function will assume -resource "aws_iam_role" "lambda_exec_role" { - name = "lambda-exec-role" - - assume_role_policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Action = "sts:AssumeRole", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - } - } - ] - }) -} - -# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging) -resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" -} - -# Custom policy: SQS access + ECR image pull permissions -resource "aws_iam_policy" "lambda_custom_policy" { - name = "lambda-sqs-ecr-policy" - - policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - # Allow Lambda to read from SQS - { - Effect = "Allow", - Action = [ - "sqs:ReceiveMessage", - "sqs:DeleteMessage", - "sqs:GetQueueAttributes" - ], - Resource = aws_sqs_queue.my_queue.arn - }, - # Allow Lambda to pull images from ECR - { - Effect = "Allow", - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ], - Resource = aws_ecr_repository.lambda_repo.arn - }, - # Needed to authenticate to ECR (pulling the image) - { - Effect = "Allow", - Action = [ - "ecr:GetAuthorizationToken" - ], - Resource = "*" - } - ] - }) -} - -# Attach the custom policy to the Lambda role -resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = aws_iam_policy.lambda_custom_policy.arn -} - -# Define the Lambda function using a Docker image from ECR -resource "aws_lambda_function" "lambda_docker" { - function_name = "docker-hello-world-python-example" - role = aws_iam_role.lambda_exec_role.arn - package_type = "Image" - image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest" - timeout = 10 -} - -# Connect the SQS queue to the Lambda so it gets triggered by incoming messages -resource "aws_lambda_event_source_mapping" "sqs_trigger" { - event_source_arn = aws_sqs_queue.my_queue.arn - function_name = aws_lambda_function.lambda_docker.arn - batch_size = 1 -} - - -resource "aws_ecr_repository_policy" "lambda_ecr_access" { - repository = aws_ecr_repository.lambda_repo.name - - policy = jsonencode({ - Version = "2008-10-17", - Statement = [ - { - Sid = "AllowLambdaPull", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - }, - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ] - } - ] - }) -} \ No newline at end of file diff --git a/deployment/lambda_example_and_config.tf b/deployment/lambda_example_and_config.tf new file mode 100644 index 0000000..b020e2d --- /dev/null +++ b/deployment/lambda_example_and_config.tf @@ -0,0 +1,106 @@ +# IAM role for both Lambdas (can be shared) +resource "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [{ + Action = "sts:AssumeRole", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + } + }] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# SQS queue for lambda_example +resource "aws_sqs_queue" "lambda_example_queue" { + name = "lambda-example-queue" +} + +# ECR repo for lambda_example +resource "aws_ecr_repository" "lambda_example" { + name = "lambda_example" +} + +# Custom IAM policy specific to lambda_example +resource "aws_iam_policy" "lambda_example_policy" { + name = "lambda-example-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.lambda_example_queue.arn + }, + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = aws_ecr_repository.lambda_example.arn + }, + { + Effect = "Allow", + Action = ["ecr:GetAuthorizationToken"], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.lambda_example_policy.arn +} + +# Lambda function +resource "aws_lambda_function" "lambda_example" { + function_name = "lambda-example" + role = aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${aws_ecr_repository.lambda_example.repository_url}:latest" + timeout = 10 +} + +# SQS trigger +resource "aws_lambda_event_source_mapping" "lambda_example_trigger" { + event_source_arn = aws_sqs_queue.lambda_example_queue.arn + function_name = aws_lambda_function.lambda_example.arn + batch_size = 1 +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "lambda_example_ecr_access" { + repository = aws_ecr_repository.lambda_example.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +} From 07c9f03c980f0ddf3e0a450dff9a05104cf9ba9a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 15:51:37 +0000 Subject: [PATCH 06/93] re build etl --- deployment/extractor_and_loader/Dockerfile | 27 +++++-- deployment/extractor_and_loader/app.py | 3 +- deployment/extractor_and_loader_lambda.tf | 4 ++ deployment/lambda_example/app.py | 1 - etl/read_stuff_from_s3_example.py | 4 ++ poetry.lock | 82 ++++++++++++++++++---- pyproject.toml | 2 +- 7 files changed, 103 insertions(+), 20 deletions(-) diff --git a/deployment/extractor_and_loader/Dockerfile b/deployment/extractor_and_loader/Dockerfile index c1caf03..609981a 100644 --- a/deployment/extractor_and_loader/Dockerfile +++ b/deployment/extractor_and_loader/Dockerfile @@ -1,8 +1,25 @@ -# AWS Lambda python pacakge -FROM public.ecr.aws/lambda/python:3.11 +FROM public.ecr.aws/lambda/python:3.12 -# Copy function code -COPY deployment/lambda_example/app.py ./ +# Install Poetry (you could pin a version if you like) +RUN curl -sSL https://install.python-poetry.org | python3 - -# Set the CMD to your handler +# Add Poetry to PATH +ENV PATH="/root/.local/bin:$PATH" + +# Set working directory +WORKDIR /var/task + +# Copy Poetry files first to leverage Docker layer caching +COPY pyproject.toml poetry.lock README.md ./ +COPY etl/ etl/ + + +# Install dependencies into /var/task +RUN poetry config virtualenvs.create false \ + && poetry install --only main --no-interaction --no-ansi + +# Copy app code +COPY deployment/extractor_and_loader/app.py ./ + +# Set Lambda handler CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py index a520464..08cb219 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/extractor_and_loader/app.py @@ -1,6 +1,7 @@ """ A quick example of lambda working a function in python """ +from etl.read_stuff_from_s3_example import print_hello_from_etl_module def handler(event, context): try: @@ -10,7 +11,7 @@ def handler(event, context): "statusCode": 400, "body": "Missing 'file_location' in event" } - + print_hello_from_etl_module() print(f"s3 uri is {s3_uri}") return { diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/extractor_and_loader_lambda.tf index 1d74b3f..9691f5f 100644 --- a/deployment/extractor_and_loader_lambda.tf +++ b/deployment/extractor_and_loader_lambda.tf @@ -83,3 +83,7 @@ resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { }] }) } + + +# TODO: Seperate lambda jobs from ecr creation. This is because we need to +# Create the ecr, then push images, then lambda jobs can be made \ No newline at end of file diff --git a/deployment/lambda_example/app.py b/deployment/lambda_example/app.py index a520464..390fa63 100644 --- a/deployment/lambda_example/app.py +++ b/deployment/lambda_example/app.py @@ -10,7 +10,6 @@ def handler(event, context): "statusCode": 400, "body": "Missing 'file_location' in event" } - print(f"s3 uri is {s3_uri}") return { diff --git a/etl/read_stuff_from_s3_example.py b/etl/read_stuff_from_s3_example.py index cb427ba..7e92635 100644 --- a/etl/read_stuff_from_s3_example.py +++ b/etl/read_stuff_from_s3_example.py @@ -1,6 +1,10 @@ import boto3 import os + +def print_hello_from_etl_module(): + print("You are printing from a etl module we made in poetry") + def split_s3_url(s3_url): if not s3_url.startswith("s3://"): raise ValueError("Invalid S3 URL. Must start with 's3://'") diff --git a/poetry.lock b/poetry.lock index 0074416..6c58987 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1243,23 +1243,81 @@ dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packagin test = ["pytest", "pytest-xdist", "setuptools"] [[package]] -name = "psycopg2" +name = "psycopg2-binary" version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, - {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, - {file = "psycopg2-2.9.10-cp311-cp311-win32.whl", hash = "sha256:47c4f9875125344f4c2b870e41b6aad585901318068acd01de93f3677a6522c2"}, - {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, - {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, - {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, - {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, - {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, - {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, - {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, + {file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win32.whl", hash = "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win32.whl", hash = "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5"}, ] [[package]] @@ -2039,4 +2097,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "7f71e898554f0285d708a349589cc96fe8cfc210e7d39caf66c1dc6faae010da" +content-hash = "dfda98ea4e00851a83a2f67c231b59476d407a1e38006610722c64842976e736" diff --git a/pyproject.toml b/pyproject.toml index 84c15cd..b22d450 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ dependencies = [ "openpyxl (>=3.1.5,<4.0.0)", "fuzzywuzzy (>=0.18.0,<0.19.0)", "sqlmodel (>=0.0.24,<0.0.25)", - "psycopg2 (>=2.9.10,<3.0.0)", "pydantic-settings (>=2.8.1,<3.0.0)", "alembic (>=1.15.1,<2.0.0)", "pytest (>=8.3.5,<9.0.0)", @@ -24,6 +23,7 @@ dependencies = [ "tqdm (>=4.67.1,<5.0.0)", "hubspot-api-client (>=12.0.0,<13.0.0)", "boto3 (>=1.39.6,<2.0.0)", + "psycopg2-binary (>=2.9.10,<3.0.0)", ] [tool.poetry] From 10e4dc50e05b245a616f2fe50029a83d3b50b579 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 16:11:22 +0000 Subject: [PATCH 07/93] reupload image --- deployment/extractor_and_loader/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py index 08cb219..8de74a3 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/extractor_and_loader/app.py @@ -6,12 +6,12 @@ from etl.read_stuff_from_s3_example import print_hello_from_etl_module def handler(event, context): try: s3_uri = event.get("file_location") + print_hello_from_etl_module() if not s3_uri: return { "statusCode": 400, "body": "Missing 'file_location' in event" } - print_hello_from_etl_module() print(f"s3 uri is {s3_uri}") return { From ba84eb8bffe8e4b7dac0ff745d793c4695b7b6ab Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 16:15:35 +0000 Subject: [PATCH 08/93] reupload image --- deployment/extractor_and_loader/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py index 8de74a3..f5bacde 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/extractor_and_loader/app.py @@ -5,6 +5,7 @@ from etl.read_stuff_from_s3_example import print_hello_from_etl_module def handler(event, context): try: + print("show me something.. anything...") s3_uri = event.get("file_location") print_hello_from_etl_module() if not s3_uri: From 75b0c13a390305bed8f2927277153702b350ec41 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 16:33:09 +0000 Subject: [PATCH 09/93] update docker image --- deployment/extractor_and_loader/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py index f5bacde..0f75d76 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/extractor_and_loader/app.py @@ -4,6 +4,7 @@ A quick example of lambda working a function in python from etl.read_stuff_from_s3_example import print_hello_from_etl_module def handler(event, context): + print("Outside try statment") try: print("show me something.. anything...") s3_uri = event.get("file_location") From 11ae8d605c762fcf688cdd19438d137d5eeb2af3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 16:33:50 +0000 Subject: [PATCH 10/93] make it basic --- deployment/extractor_and_loader/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py index 0f75d76..9e5e023 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/extractor_and_loader/app.py @@ -1,14 +1,14 @@ """ A quick example of lambda working a function in python """ -from etl.read_stuff_from_s3_example import print_hello_from_etl_module +# from etl.read_stuff_from_s3_example import print_hello_from_etl_module def handler(event, context): print("Outside try statment") try: print("show me something.. anything...") s3_uri = event.get("file_location") - print_hello_from_etl_module() + # print_hello_from_etl_module() if not s3_uri: return { "statusCode": 400, From f5d63ca84441fc621fced7704e281d7a4df99c49 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 16:41:15 +0000 Subject: [PATCH 11/93] brand new image --- .github/workflows/push_docker_image_to_ecr.yml | 2 +- deployment/extractor_and_loader_lambda.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 178e646..9028e60 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -76,7 +76,7 @@ jobs: - name: Build, tag, and push Docker image to ECR env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: latest + IMAGE_TAG: latest2 run: | IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} echo "pwd" diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/extractor_and_loader_lambda.tf index 9691f5f..f8de71c 100644 --- a/deployment/extractor_and_loader_lambda.tf +++ b/deployment/extractor_and_loader_lambda.tf @@ -52,8 +52,8 @@ resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest" - timeout = 10 + image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest2" + timeout = 30 } # SQS trigger From 3133fb74be761edfaab98a1747998696a8f47ce5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 16:47:04 +0000 Subject: [PATCH 12/93] brand new image --- .github/workflows/push_docker_image_to_ecr.yml | 2 +- deployment/extractor_and_loader/app.py | 4 ++-- deployment/extractor_and_loader_lambda.tf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 9028e60..4e0ac57 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -76,7 +76,7 @@ jobs: - name: Build, tag, and push Docker image to ECR env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: latest2 + IMAGE_TAG: latest3 run: | IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} echo "pwd" diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py index 9e5e023..0f75d76 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/extractor_and_loader/app.py @@ -1,14 +1,14 @@ """ A quick example of lambda working a function in python """ -# from etl.read_stuff_from_s3_example import print_hello_from_etl_module +from etl.read_stuff_from_s3_example import print_hello_from_etl_module def handler(event, context): print("Outside try statment") try: print("show me something.. anything...") s3_uri = event.get("file_location") - # print_hello_from_etl_module() + print_hello_from_etl_module() if not s3_uri: return { "statusCode": 400, diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/extractor_and_loader_lambda.tf index f8de71c..316e212 100644 --- a/deployment/extractor_and_loader_lambda.tf +++ b/deployment/extractor_and_loader_lambda.tf @@ -52,7 +52,7 @@ resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest2" + image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest3" timeout = 30 } From 00b699ec24fc58bbd1cbc7cbfbd3956c3241d6b9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Jul 2025 17:27:11 +0000 Subject: [PATCH 13/93] save --- deployment/extractor_and_loader/app.py | 2 +- deployment/extractor_and_loader_lambda.tf | 2 +- etl/read_stuff_from_s3_example.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py index 0f75d76..a8ece33 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/extractor_and_loader/app.py @@ -5,10 +5,10 @@ from etl.read_stuff_from_s3_example import print_hello_from_etl_module def handler(event, context): print("Outside try statment") + print_hello_from_etl_module() try: print("show me something.. anything...") s3_uri = event.get("file_location") - print_hello_from_etl_module() if not s3_uri: return { "statusCode": 400, diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/extractor_and_loader_lambda.tf index 316e212..a6ac534 100644 --- a/deployment/extractor_and_loader_lambda.tf +++ b/deployment/extractor_and_loader_lambda.tf @@ -52,7 +52,7 @@ resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest3" + image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest" timeout = 30 } diff --git a/etl/read_stuff_from_s3_example.py b/etl/read_stuff_from_s3_example.py index 7e92635..6ffdbcc 100644 --- a/etl/read_stuff_from_s3_example.py +++ b/etl/read_stuff_from_s3_example.py @@ -41,4 +41,4 @@ def download_data_from_s3(s3_uri): return create_temp_file(data, file_name) # Example usage -download_data_from_s3("s3://retrofit-energy-assessments-dev/JAFFERSONS ENERGY CONSULTANTS/VDE001/12103116/docs & plans/77 Perryn Road, W3 7LT EPR.pdf") +# download_data_from_s3("s3://retrofit-energy-assessments-dev/JAFFERSONS ENERGY CONSULTANTS/VDE001/12103116/docs & plans/77 Perryn Road, W3 7LT EPR.pdf") From 53343c625e98fb5824c69bf64beb5b99033588aa Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 10:41:14 +0000 Subject: [PATCH 14/93] latest 5 --- .github/workflows/push_docker_image_to_ecr.yml | 2 +- deployment/extractor_and_loader_lambda.tf | 2 +- deployment/lambda_example_and_config.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 4e0ac57..5e790bf 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -76,7 +76,7 @@ jobs: - name: Build, tag, and push Docker image to ECR env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: latest3 + IMAGE_TAG: latest5 run: | IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} echo "pwd" diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/extractor_and_loader_lambda.tf index a6ac534..eb0aed9 100644 --- a/deployment/extractor_and_loader_lambda.tf +++ b/deployment/extractor_and_loader_lambda.tf @@ -52,7 +52,7 @@ resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest" + image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest5" timeout = 30 } diff --git a/deployment/lambda_example_and_config.tf b/deployment/lambda_example_and_config.tf index b020e2d..f0f88e5 100644 --- a/deployment/lambda_example_and_config.tf +++ b/deployment/lambda_example_and_config.tf @@ -73,7 +73,7 @@ resource "aws_lambda_function" "lambda_example" { function_name = "lambda-example" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.lambda_example.repository_url}:latest" + image_uri = "${aws_ecr_repository.lambda_example.repository_url}:latest4" timeout = 10 } From 8747e671b70e855f907eb6a464fae3ac950ab9cf Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 13:33:20 +0000 Subject: [PATCH 15/93] lambda states --- deployment/{ => database}/main.tf | 0 deployment/{ => database}/output.tf | 0 deployment/{ => database}/provider.tf | 0 deployment/{ => database}/variables.tf | 0 deployment/{ => database}/vpc.tf | 0 deployment/{ => lambda}/extractor_and_loader/Dockerfile | 0 deployment/{ => lambda}/extractor_and_loader/app.py | 1 + .../extractor_and_loader}/extractor_and_loader_lambda.tf | 0 deployment/{ => lambda}/lambda_example/Dockerfile | 0 deployment/{ => lambda}/lambda_example/app.py | 0 .../{ => lambda/lambda_example}/lambda_example_and_config.tf | 2 +- 11 files changed, 2 insertions(+), 1 deletion(-) rename deployment/{ => database}/main.tf (100%) rename deployment/{ => database}/output.tf (100%) rename deployment/{ => database}/provider.tf (100%) rename deployment/{ => database}/variables.tf (100%) rename deployment/{ => database}/vpc.tf (100%) rename deployment/{ => lambda}/extractor_and_loader/Dockerfile (100%) rename deployment/{ => lambda}/extractor_and_loader/app.py (94%) rename deployment/{ => lambda/extractor_and_loader}/extractor_and_loader_lambda.tf (100%) rename deployment/{ => lambda}/lambda_example/Dockerfile (100%) rename deployment/{ => lambda}/lambda_example/app.py (100%) rename deployment/{ => lambda/lambda_example}/lambda_example_and_config.tf (99%) diff --git a/deployment/main.tf b/deployment/database/main.tf similarity index 100% rename from deployment/main.tf rename to deployment/database/main.tf diff --git a/deployment/output.tf b/deployment/database/output.tf similarity index 100% rename from deployment/output.tf rename to deployment/database/output.tf diff --git a/deployment/provider.tf b/deployment/database/provider.tf similarity index 100% rename from deployment/provider.tf rename to deployment/database/provider.tf diff --git a/deployment/variables.tf b/deployment/database/variables.tf similarity index 100% rename from deployment/variables.tf rename to deployment/database/variables.tf diff --git a/deployment/vpc.tf b/deployment/database/vpc.tf similarity index 100% rename from deployment/vpc.tf rename to deployment/database/vpc.tf diff --git a/deployment/extractor_and_loader/Dockerfile b/deployment/lambda/extractor_and_loader/Dockerfile similarity index 100% rename from deployment/extractor_and_loader/Dockerfile rename to deployment/lambda/extractor_and_loader/Dockerfile diff --git a/deployment/extractor_and_loader/app.py b/deployment/lambda/extractor_and_loader/app.py similarity index 94% rename from deployment/extractor_and_loader/app.py rename to deployment/lambda/extractor_and_loader/app.py index a8ece33..4a281ed 100644 --- a/deployment/extractor_and_loader/app.py +++ b/deployment/lambda/extractor_and_loader/app.py @@ -10,6 +10,7 @@ def handler(event, context): print("show me something.. anything...") s3_uri = event.get("file_location") if not s3_uri: + print("failed to get s3_uri") return { "statusCode": 400, "body": "Missing 'file_location' in event" diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf similarity index 100% rename from deployment/extractor_and_loader_lambda.tf rename to deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf diff --git a/deployment/lambda_example/Dockerfile b/deployment/lambda/lambda_example/Dockerfile similarity index 100% rename from deployment/lambda_example/Dockerfile rename to deployment/lambda/lambda_example/Dockerfile diff --git a/deployment/lambda_example/app.py b/deployment/lambda/lambda_example/app.py similarity index 100% rename from deployment/lambda_example/app.py rename to deployment/lambda/lambda_example/app.py diff --git a/deployment/lambda_example_and_config.tf b/deployment/lambda/lambda_example/lambda_example_and_config.tf similarity index 99% rename from deployment/lambda_example_and_config.tf rename to deployment/lambda/lambda_example/lambda_example_and_config.tf index f0f88e5..b020e2d 100644 --- a/deployment/lambda_example_and_config.tf +++ b/deployment/lambda/lambda_example/lambda_example_and_config.tf @@ -73,7 +73,7 @@ resource "aws_lambda_function" "lambda_example" { function_name = "lambda-example" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.lambda_example.repository_url}:latest4" + image_uri = "${aws_ecr_repository.lambda_example.repository_url}:latest" timeout = 10 } From 1db94642ae8eb3b7cdd966a44dcf5c41307bfd63 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 13:58:12 +0000 Subject: [PATCH 16/93] save change here --- deployment/lambda/lambda_shared/lambda_shared_condig.tf | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 deployment/lambda/lambda_shared/lambda_shared_condig.tf diff --git a/deployment/lambda/lambda_shared/lambda_shared_condig.tf b/deployment/lambda/lambda_shared/lambda_shared_condig.tf new file mode 100644 index 0000000..e69de29 From 70a81feb0c6db7b878131c5b9b8576f97826ca6a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 15:09:53 +0000 Subject: [PATCH 17/93] shared lambda complete --- deployment/database/provider.tf | 4 +- .../lambda/extractor_and_loader/Dockerfile | 25 ---------- deployment/lambda/extractor_and_loader/app.py | 30 ------------ deployment/lambda/lambda_example/Dockerfile | 8 ---- deployment/lambda/lambda_example/app.py | 25 ---------- .../lambda_example_and_config.tf | 46 +------------------ .../lambda_shared/lambda_shared_config.tf | 21 +++++++++ .../{lambda_shared_condig.tf => main.tf} | 0 deployment/lambda/lambda_shared/provider.tf | 16 +++++++ 9 files changed, 40 insertions(+), 135 deletions(-) delete mode 100644 deployment/lambda/extractor_and_loader/Dockerfile delete mode 100644 deployment/lambda/extractor_and_loader/app.py delete mode 100644 deployment/lambda/lambda_example/Dockerfile delete mode 100644 deployment/lambda/lambda_example/app.py create mode 100644 deployment/lambda/lambda_shared/lambda_shared_config.tf rename deployment/lambda/lambda_shared/{lambda_shared_condig.tf => main.tf} (100%) create mode 100644 deployment/lambda/lambda_shared/provider.tf diff --git a/deployment/database/provider.tf b/deployment/database/provider.tf index 46e887f..009e591 100644 --- a/deployment/database/provider.tf +++ b/deployment/database/provider.tf @@ -2,14 +2,14 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = "~> 6.3.0" } } backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" profile = "domna.dev" # /home/vscode/aws/credentials - key = "terraform.tfstate" + key = "env:/dev/terraform.tfstate" } required_version = ">= 1.2.0" diff --git a/deployment/lambda/extractor_and_loader/Dockerfile b/deployment/lambda/extractor_and_loader/Dockerfile deleted file mode 100644 index 609981a..0000000 --- a/deployment/lambda/extractor_and_loader/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.12 - -# Install Poetry (you could pin a version if you like) -RUN curl -sSL https://install.python-poetry.org | python3 - - -# Add Poetry to PATH -ENV PATH="/root/.local/bin:$PATH" - -# Set working directory -WORKDIR /var/task - -# Copy Poetry files first to leverage Docker layer caching -COPY pyproject.toml poetry.lock README.md ./ -COPY etl/ etl/ - - -# Install dependencies into /var/task -RUN poetry config virtualenvs.create false \ - && poetry install --only main --no-interaction --no-ansi - -# Copy app code -COPY deployment/extractor_and_loader/app.py ./ - -# Set Lambda handler -CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/app.py b/deployment/lambda/extractor_and_loader/app.py deleted file mode 100644 index 4a281ed..0000000 --- a/deployment/lambda/extractor_and_loader/app.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -A quick example of lambda working a function in python -""" -from etl.read_stuff_from_s3_example import print_hello_from_etl_module - -def handler(event, context): - print("Outside try statment") - print_hello_from_etl_module() - try: - print("show me something.. anything...") - s3_uri = event.get("file_location") - if not s3_uri: - print("failed to get s3_uri") - return { - "statusCode": 400, - "body": "Missing 'file_location' in event" - } - print(f"s3 uri is {s3_uri}") - - return { - "statusCode": 200, - "body": f"s3 uri {s3_uri}" - } - - except Exception as e: - print(f"❌ Error: {e}") - return { - "statusCode": 500, - "body": str(e) - } \ No newline at end of file diff --git a/deployment/lambda/lambda_example/Dockerfile b/deployment/lambda/lambda_example/Dockerfile deleted file mode 100644 index c1caf03..0000000 --- a/deployment/lambda/lambda_example/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -# AWS Lambda python pacakge -FROM public.ecr.aws/lambda/python:3.11 - -# Copy function code -COPY deployment/lambda_example/app.py ./ - -# Set the CMD to your handler -CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/lambda_example/app.py b/deployment/lambda/lambda_example/app.py deleted file mode 100644 index 390fa63..0000000 --- a/deployment/lambda/lambda_example/app.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -A quick example of lambda working a function in python -""" - -def handler(event, context): - try: - s3_uri = event.get("file_location") - if not s3_uri: - return { - "statusCode": 400, - "body": "Missing 'file_location' in event" - } - print(f"s3 uri is {s3_uri}") - - return { - "statusCode": 200, - "body": f"s3 uri {s3_uri}" - } - - except Exception as e: - print(f"❌ Error: {e}") - return { - "statusCode": 500, - "body": str(e) - } \ No newline at end of file diff --git a/deployment/lambda/lambda_example/lambda_example_and_config.tf b/deployment/lambda/lambda_example/lambda_example_and_config.tf index b020e2d..581d038 100644 --- a/deployment/lambda/lambda_example/lambda_example_and_config.tf +++ b/deployment/lambda/lambda_example/lambda_example_and_config.tf @@ -1,33 +1,9 @@ -# IAM role for both Lambdas (can be shared) -resource "aws_iam_role" "lambda_exec_role" { - name = "lambda-exec-role" - - assume_role_policy = jsonencode({ - Version = "2012-10-17", - Statement = [{ - Action = "sts:AssumeRole", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - } - }] - }) -} - -resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" -} - # SQS queue for lambda_example resource "aws_sqs_queue" "lambda_example_queue" { name = "lambda-example-queue" } -# ECR repo for lambda_example -resource "aws_ecr_repository" "lambda_example" { - name = "lambda_example" -} + # Custom IAM policy specific to lambda_example resource "aws_iam_policy" "lambda_example_policy" { @@ -84,23 +60,3 @@ resource "aws_lambda_event_source_mapping" "lambda_example_trigger" { batch_size = 1 } -# ECR policy to allow Lambda access -resource "aws_ecr_repository_policy" "lambda_example_ecr_access" { - repository = aws_ecr_repository.lambda_example.name - - policy = jsonencode({ - Version = "2008-10-17", - Statement = [{ - Sid = "AllowLambdaPull", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - }, - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ] - }] - }) -} diff --git a/deployment/lambda/lambda_shared/lambda_shared_config.tf b/deployment/lambda/lambda_shared/lambda_shared_config.tf new file mode 100644 index 0000000..1533dfb --- /dev/null +++ b/deployment/lambda/lambda_shared/lambda_shared_config.tf @@ -0,0 +1,21 @@ +# IAM role for both Lambdas (can be shared) +resource "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [{ + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = "sts:AssumeRole" + }] + }) +} + + +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} \ No newline at end of file diff --git a/deployment/lambda/lambda_shared/lambda_shared_condig.tf b/deployment/lambda/lambda_shared/main.tf similarity index 100% rename from deployment/lambda/lambda_shared/lambda_shared_condig.tf rename to deployment/lambda/lambda_shared/main.tf diff --git a/deployment/lambda/lambda_shared/provider.tf b/deployment/lambda/lambda_shared/provider.tf new file mode 100644 index 0000000..fa6a5d4 --- /dev/null +++ b/deployment/lambda/lambda_shared/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + profile = "domna.dev" # /home/vscode/aws/credentials + key = "env:/dev/lambda/lambda_share_configuration.tfstate" + } + + required_version = ">= 1.2.0" +} From 8948e64e136a409bbdc132a3f75bee699110582b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 15:12:43 +0000 Subject: [PATCH 18/93] added docker for lambda example --- .../lambda/lambda_example/docker/Dockerfile | 8 ++++++ .../lambda/lambda_example/docker/app.py | 25 +++++++++++++++++++ .../lambda/lambda_example/docker/ecr.tf | 25 +++++++++++++++++++ .../lambda/lambda_example/docker/main.tf | 0 .../lambda/lambda_example/docker/provider.tf | 16 ++++++++++++ 5 files changed, 74 insertions(+) create mode 100644 deployment/lambda/lambda_example/docker/Dockerfile create mode 100644 deployment/lambda/lambda_example/docker/app.py create mode 100644 deployment/lambda/lambda_example/docker/ecr.tf create mode 100644 deployment/lambda/lambda_example/docker/main.tf create mode 100644 deployment/lambda/lambda_example/docker/provider.tf diff --git a/deployment/lambda/lambda_example/docker/Dockerfile b/deployment/lambda/lambda_example/docker/Dockerfile new file mode 100644 index 0000000..c1caf03 --- /dev/null +++ b/deployment/lambda/lambda_example/docker/Dockerfile @@ -0,0 +1,8 @@ +# AWS Lambda python pacakge +FROM public.ecr.aws/lambda/python:3.11 + +# Copy function code +COPY deployment/lambda_example/app.py ./ + +# Set the CMD to your handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/lambda_example/docker/app.py b/deployment/lambda/lambda_example/docker/app.py new file mode 100644 index 0000000..390fa63 --- /dev/null +++ b/deployment/lambda/lambda_example/docker/app.py @@ -0,0 +1,25 @@ +""" +A quick example of lambda working a function in python +""" + +def handler(event, context): + try: + s3_uri = event.get("file_location") + if not s3_uri: + return { + "statusCode": 400, + "body": "Missing 'file_location' in event" + } + print(f"s3 uri is {s3_uri}") + + return { + "statusCode": 200, + "body": f"s3 uri {s3_uri}" + } + + except Exception as e: + print(f"❌ Error: {e}") + return { + "statusCode": 500, + "body": str(e) + } \ No newline at end of file diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf new file mode 100644 index 0000000..d9e2eab --- /dev/null +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -0,0 +1,25 @@ +# ECR repo for lambda_example +resource "aws_ecr_repository" "lambda_example" { + name = "lambda_example" +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "lambda_example_ecr_access" { + repository = aws_ecr_repository.lambda_example.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +} diff --git a/deployment/lambda/lambda_example/docker/main.tf b/deployment/lambda/lambda_example/docker/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/lambda_example/docker/provider.tf b/deployment/lambda/lambda_example/docker/provider.tf new file mode 100644 index 0000000..fca37f7 --- /dev/null +++ b/deployment/lambda/lambda_example/docker/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + profile = "domna.dev" # /home/vscode/aws/credentials + key = "env:/dev/lambda/ecr/lambda_example_ecr.tfstate" + } + + required_version = ">= 1.2.0" +} From ae182bb81f316eaf2064475693d1f014dedf25a1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 15:16:02 +0000 Subject: [PATCH 19/93] added github run --- .../workflows/push_docker_image_to_ecr.yml | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 5e790bf..057ca91 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -2,7 +2,7 @@ name: Build and Push Docker Image to ECR lambda example on: push: - branches: [feature/elmhurst_epr, main] + branches: [feature/seperate_terraform_with_different_states, main] env: AWS_REGION: eu-west-2 @@ -48,42 +48,42 @@ jobs: echo "Pushing Docker image to ECR..." docker push $IMAGE_URI - build-and-push-to-ecr-for-extractor-and-loader-example: - runs-on: ubuntu-latest - env: - ECR_REPOSITORY: extractor_and_loader + # build-and-push-to-ecr-for-extractor-and-loader-example: + # runs-on: ubuntu-latest + # env: + # ECR_REPOSITORY: extractor_and_loader - permissions: - id-token: write - contents: read + # permissions: + # id-token: write + # contents: read - steps: - - name: Checkout code - uses: actions/checkout@v4 + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 - - name: AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # as of 14/07/2025 it'll be using user:Junte's keys - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + # - name: AWS credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # # as of 14/07/2025 it'll be using user:Junte's keys + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} - - name: Log in to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + # - name: Log in to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v2 - - name: Build, tag, and push Docker image to ECR - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: latest5 - run: | - IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - echo "pwd" - pwd - ls -la - echo "Building Docker image..." - docker build -t $IMAGE_URI -f deployment/extractor_and_loader/Dockerfile . + # - name: Build, tag, and push Docker image to ECR + # env: + # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # IMAGE_TAG: latest5 + # run: | + # IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + # echo "pwd" + # pwd + # ls -la + # echo "Building Docker image..." + # docker build -t $IMAGE_URI -f deployment/extractor_and_loader/Dockerfile . - echo "Pushing Docker image to ECR..." - docker push $IMAGE_URI \ No newline at end of file + # echo "Pushing Docker image to ECR..." + # docker push $IMAGE_URI \ No newline at end of file From 4687c5098ed820f050dfe2d277a0b1238a7aa117 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 15:18:46 +0000 Subject: [PATCH 20/93] location ficxed --- .github/workflows/push_docker_image_to_ecr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 057ca91..cf96382 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -43,7 +43,7 @@ jobs: pwd ls -la echo "Building Docker image..." - docker build -t $IMAGE_URI -f deployment/lambda_example/Dockerfile . + docker build -t $IMAGE_URI -f deployment/lambda/lambda_example/docker/Dockerfile . echo "Pushing Docker image to ECR..." docker push $IMAGE_URI From 0f66ff70d9b421793ab872e750f6e3375887f7d0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 15:20:29 +0000 Subject: [PATCH 21/93] location ficxed --- deployment/lambda/lambda_example/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/lambda/lambda_example/docker/Dockerfile b/deployment/lambda/lambda_example/docker/Dockerfile index c1caf03..3aa661b 100644 --- a/deployment/lambda/lambda_example/docker/Dockerfile +++ b/deployment/lambda/lambda_example/docker/Dockerfile @@ -2,7 +2,7 @@ FROM public.ecr.aws/lambda/python:3.11 # Copy function code -COPY deployment/lambda_example/app.py ./ +COPY deployment/lambda/lambda_example/docker/app.py ./ # Set the CMD to your handler CMD ["app.handler"] \ No newline at end of file From 0be7ef6c89e9f09bb58863c5665e442cf7c48649 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 15:56:37 +0000 Subject: [PATCH 22/93] lambda example completed --- .../lambda/lambda_example/docker/output.tf | 3 +++ .../lambda_example_and_config.tf | 21 ++++++++++++------- deployment/lambda/lambda_example/main.tf | 0 deployment/lambda/lambda_example/provider.tf | 16 ++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 deployment/lambda/lambda_example/docker/output.tf create mode 100644 deployment/lambda/lambda_example/main.tf create mode 100644 deployment/lambda/lambda_example/provider.tf diff --git a/deployment/lambda/lambda_example/docker/output.tf b/deployment/lambda/lambda_example/docker/output.tf new file mode 100644 index 0000000..1b797cf --- /dev/null +++ b/deployment/lambda/lambda_example/docker/output.tf @@ -0,0 +1,3 @@ +output "ecr_repo_url" { + value = aws_ecr_repository.lambda_example.repository_url +} \ No newline at end of file diff --git a/deployment/lambda/lambda_example/lambda_example_and_config.tf b/deployment/lambda/lambda_example/lambda_example_and_config.tf index 581d038..1fbd5ad 100644 --- a/deployment/lambda/lambda_example/lambda_example_and_config.tf +++ b/deployment/lambda/lambda_example/lambda_example_and_config.tf @@ -1,10 +1,18 @@ +# Reference existing IAM role +data "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" +} + +# Reference existing ECR repository +data "aws_ecr_repository" "lambda_example" { + name = "lambda_example" +} + # SQS queue for lambda_example resource "aws_sqs_queue" "lambda_example_queue" { name = "lambda-example-queue" } - - # Custom IAM policy specific to lambda_example resource "aws_iam_policy" "lambda_example_policy" { name = "lambda-example-policy" @@ -28,7 +36,7 @@ resource "aws_iam_policy" "lambda_example_policy" { "ecr:BatchGetImage", "ecr:BatchCheckLayerAvailability" ], - Resource = aws_ecr_repository.lambda_example.arn + Resource = data.aws_ecr_repository.lambda_example.arn }, { Effect = "Allow", @@ -40,16 +48,16 @@ resource "aws_iam_policy" "lambda_example_policy" { } resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" { - role = aws_iam_role.lambda_exec_role.name + role = data.aws_iam_role.lambda_exec_role.name policy_arn = aws_iam_policy.lambda_example_policy.arn } # Lambda function resource "aws_lambda_function" "lambda_example" { function_name = "lambda-example" - role = aws_iam_role.lambda_exec_role.arn + role = data.aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.lambda_example.repository_url}:latest" + image_uri = "${data.aws_ecr_repository.lambda_example.repository_url}:latest" timeout = 10 } @@ -59,4 +67,3 @@ resource "aws_lambda_event_source_mapping" "lambda_example_trigger" { function_name = aws_lambda_function.lambda_example.arn batch_size = 1 } - diff --git a/deployment/lambda/lambda_example/main.tf b/deployment/lambda/lambda_example/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/lambda_example/provider.tf b/deployment/lambda/lambda_example/provider.tf new file mode 100644 index 0000000..bac0dad --- /dev/null +++ b/deployment/lambda/lambda_example/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + profile = "domna.dev" # /home/vscode/aws/credentials + key = "env:/dev/lambda/eachlambda/lambda_example.tfstate" + } + + required_version = ">= 1.2.0" +} From e5ba53e787818fa23ca7dce3ee83945dd7bf121b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Jul 2025 20:18:46 +0000 Subject: [PATCH 23/93] save exisiting work --- .../extractor_and_loader/docker/Dockerfile | 25 ++++++++++++++++ .../lambda/extractor_and_loader/docker/app.py | 30 +++++++++++++++++++ .../extractor_and_loader/docker/main.tf | 0 .../lambda/extractor_and_loader/main.tf | 0 .../lambda/extractor_and_loader/provider.tf | 16 ++++++++++ deployment/lambda/lambda_shared/output.tf | 9 ++++++ 6 files changed, 80 insertions(+) create mode 100644 deployment/lambda/extractor_and_loader/docker/Dockerfile create mode 100644 deployment/lambda/extractor_and_loader/docker/app.py create mode 100644 deployment/lambda/extractor_and_loader/docker/main.tf create mode 100644 deployment/lambda/extractor_and_loader/main.tf create mode 100644 deployment/lambda/extractor_and_loader/provider.tf create mode 100644 deployment/lambda/lambda_shared/output.tf diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile new file mode 100644 index 0000000..609981a --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -0,0 +1,25 @@ +FROM public.ecr.aws/lambda/python:3.12 + +# Install Poetry (you could pin a version if you like) +RUN curl -sSL https://install.python-poetry.org | python3 - + +# Add Poetry to PATH +ENV PATH="/root/.local/bin:$PATH" + +# Set working directory +WORKDIR /var/task + +# Copy Poetry files first to leverage Docker layer caching +COPY pyproject.toml poetry.lock README.md ./ +COPY etl/ etl/ + + +# Install dependencies into /var/task +RUN poetry config virtualenvs.create false \ + && poetry install --only main --no-interaction --no-ansi + +# Copy app code +COPY deployment/extractor_and_loader/app.py ./ + +# Set Lambda handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py new file mode 100644 index 0000000..4a281ed --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -0,0 +1,30 @@ +""" +A quick example of lambda working a function in python +""" +from etl.read_stuff_from_s3_example import print_hello_from_etl_module + +def handler(event, context): + print("Outside try statment") + print_hello_from_etl_module() + try: + print("show me something.. anything...") + s3_uri = event.get("file_location") + if not s3_uri: + print("failed to get s3_uri") + return { + "statusCode": 400, + "body": "Missing 'file_location' in event" + } + print(f"s3 uri is {s3_uri}") + + return { + "statusCode": 200, + "body": f"s3 uri {s3_uri}" + } + + except Exception as e: + print(f"❌ Error: {e}") + return { + "statusCode": 500, + "body": str(e) + } \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/main.tf b/deployment/lambda/extractor_and_loader/docker/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/extractor_and_loader/main.tf b/deployment/lambda/extractor_and_loader/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/extractor_and_loader/provider.tf b/deployment/lambda/extractor_and_loader/provider.tf new file mode 100644 index 0000000..0997e35 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + profile = "domna.dev" # /home/vscode/aws/credentials + key = "env:/dev/extractor_and_loader_lambda.tfstate" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/lambda/lambda_shared/output.tf b/deployment/lambda/lambda_shared/output.tf new file mode 100644 index 0000000..9d4094d --- /dev/null +++ b/deployment/lambda/lambda_shared/output.tf @@ -0,0 +1,9 @@ +output "lambda_exec_role_arn" { + description = "The ARN of the IAM role used by the Lambda functions" + value = aws_iam_role.lambda_exec_role.arn +} + +output "lambda_exec_role_name" { + description = "The ARN of the IAM role used by the Lambda functions" + value = aws_iam_role.lambda_exec_role.name +} From cf2f4584c0a47bf46e973857ada4aa45c4f3f839 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 09:45:43 +0000 Subject: [PATCH 24/93] deploy image --- .../workflows/push_docker_image_to_ecr.yml | 66 +++++++++---------- .../extractor_and_loader/docker/Dockerfile | 2 +- .../extractor_and_loader_lambda.tf | 41 ++++-------- 3 files changed, 47 insertions(+), 62 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index cf96382..d96507f 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -48,42 +48,42 @@ jobs: echo "Pushing Docker image to ECR..." docker push $IMAGE_URI - # build-and-push-to-ecr-for-extractor-and-loader-example: - # runs-on: ubuntu-latest - # env: - # ECR_REPOSITORY: extractor_and_loader + build-and-push-to-ecr-for-extractor-and-loader-example: + runs-on: ubuntu-latest + env: + ECR_REPOSITORY: extractor_and_loader - # permissions: - # id-token: write - # contents: read + permissions: + id-token: write + contents: read - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 + steps: + - name: Checkout code + uses: actions/checkout@v4 - # - name: AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # # as of 14/07/2025 it'll be using user:Junte's keys - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.AWS_REGION }} + - name: AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # as of 14/07/2025 it'll be using user:Junte's keys + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} - # - name: Log in to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v2 + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 - # - name: Build, tag, and push Docker image to ECR - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # IMAGE_TAG: latest5 - # run: | - # IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - # echo "pwd" - # pwd - # ls -la - # echo "Building Docker image..." - # docker build -t $IMAGE_URI -f deployment/extractor_and_loader/Dockerfile . + - name: Build, tag, and push Docker image to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: latest + run: | + IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + echo "pwd" + pwd + ls -la + echo "Building Docker image..." + docker build -t $IMAGE_URI -f deployment/lambda/extractor_and_loader/docker/Dockerfile . - # echo "Pushing Docker image to ECR..." - # docker push $IMAGE_URI \ No newline at end of file + echo "Pushing Docker image to ECR..." + docker push $IMAGE_URI \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index 609981a..cdd1f8a 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -19,7 +19,7 @@ RUN poetry config virtualenvs.create false \ && poetry install --only main --no-interaction --no-ansi # Copy app code -COPY deployment/extractor_and_loader/app.py ./ +COPY deployment/lambda/extractor_and_loader/docker/app.py ./ # Set Lambda handler CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf index eb0aed9..71ff4d1 100644 --- a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf +++ b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf @@ -1,12 +1,18 @@ +# Reference existing IAM role +data "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" +} + +# Reference existing ECR repository +data "aws_ecr_repository" "extractor_and_loader" { + name = "extractor_and_loader" +} + # SQS queue for extractor_and_loader resource "aws_sqs_queue" "extractor_and_loader_queue" { name = "extractor-loader-queue" } -# ECR repo -resource "aws_ecr_repository" "extractor_and_loader" { - name = "extractor_and_loader" -} # IAM policy specific to this Lambda resource "aws_iam_policy" "extractor_loader_policy" { @@ -31,7 +37,7 @@ resource "aws_iam_policy" "extractor_loader_policy" { "ecr:BatchGetImage", "ecr:BatchCheckLayerAvailability" ], - Resource = aws_ecr_repository.extractor_and_loader.arn + Resource = data.aws_ecr_repository.lambda_example.arn }, { Effect = "Allow", @@ -43,7 +49,7 @@ resource "aws_iam_policy" "extractor_loader_policy" { } resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" { - role = aws_iam_role.lambda_exec_role.name + role = data.aws_iam_role.lambda_exec_role.name policy_arn = aws_iam_policy.extractor_loader_policy.arn } @@ -52,7 +58,7 @@ resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest5" + mage_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:latest" timeout = 30 } @@ -63,27 +69,6 @@ resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" { batch_size = 1 } -# ECR policy to allow Lambda access -resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { - repository = aws_ecr_repository.extractor_and_loader.name - - policy = jsonencode({ - Version = "2008-10-17", - Statement = [{ - Sid = "AllowLambdaPull", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - }, - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ] - }] - }) -} - # TODO: Seperate lambda jobs from ecr creation. This is because we need to # Create the ecr, then push images, then lambda jobs can be made \ No newline at end of file From 9a914a6e978cd2e2a5c9af5c88157afd87844e01 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:10:16 +0000 Subject: [PATCH 25/93] added extractor and loader --- .../lambda/extractor_and_loader/docker/ecr.tf | 26 +++++++++++++++++++ .../extractor_and_loader/docker/provider.tf | 16 ++++++++++++ .../extractor_and_loader_lambda.tf | 6 ++--- .../lambda/extractor_and_loader/provider.tf | 2 +- .../lambda/lambda_example/docker/app.py | 8 +----- 5 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 deployment/lambda/extractor_and_loader/docker/ecr.tf create mode 100644 deployment/lambda/extractor_and_loader/docker/provider.tf diff --git a/deployment/lambda/extractor_and_loader/docker/ecr.tf b/deployment/lambda/extractor_and_loader/docker/ecr.tf new file mode 100644 index 0000000..6b2f56a --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/ecr.tf @@ -0,0 +1,26 @@ +# ECR repo +resource "aws_ecr_repository" "extractor_and_loader" { + name = "extractor_and_loader" +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { + repository = aws_ecr_repository.extractor_and_loader.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +} + diff --git a/deployment/lambda/extractor_and_loader/docker/provider.tf b/deployment/lambda/extractor_and_loader/docker/provider.tf new file mode 100644 index 0000000..d8a8f1f --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + profile = "domna.dev" # /home/vscode/aws/credentials + key = "env:/dev/lambda/ecr/extractor_and_loader.tfstate" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf index 71ff4d1..91b5755 100644 --- a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf +++ b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf @@ -37,7 +37,7 @@ resource "aws_iam_policy" "extractor_loader_policy" { "ecr:BatchGetImage", "ecr:BatchCheckLayerAvailability" ], - Resource = data.aws_ecr_repository.lambda_example.arn + Resource = data.aws_ecr_repository.extractor_and_loader.arn }, { Effect = "Allow", @@ -56,9 +56,9 @@ resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" { # Lambda function resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" - role = aws_iam_role.lambda_exec_role.arn + role = data.aws_iam_role.lambda_exec_role.arn package_type = "Image" - mage_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:latest" + image_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:latest" timeout = 30 } diff --git a/deployment/lambda/extractor_and_loader/provider.tf b/deployment/lambda/extractor_and_loader/provider.tf index 0997e35..c5008c4 100644 --- a/deployment/lambda/extractor_and_loader/provider.tf +++ b/deployment/lambda/extractor_and_loader/provider.tf @@ -9,7 +9,7 @@ terraform { bucket = "survey-extractor-tf-state" region = "eu-west-2" profile = "domna.dev" # /home/vscode/aws/credentials - key = "env:/dev/extractor_and_loader_lambda.tfstate" + key = "env:/dev/lambda/eachlambda/extractor_and_loader_lambda.tfstate" } required_version = ">= 1.2.0" diff --git a/deployment/lambda/lambda_example/docker/app.py b/deployment/lambda/lambda_example/docker/app.py index 390fa63..4bd6ec9 100644 --- a/deployment/lambda/lambda_example/docker/app.py +++ b/deployment/lambda/lambda_example/docker/app.py @@ -4,13 +4,7 @@ A quick example of lambda working a function in python def handler(event, context): try: - s3_uri = event.get("file_location") - if not s3_uri: - return { - "statusCode": 400, - "body": "Missing 'file_location' in event" - } - print(f"s3 uri is {s3_uri}") + print("Printing from lambda example") return { "statusCode": 200, From f7540729223ea065d3bc5d0ccd5e0cbd0a687603 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:22:52 +0000 Subject: [PATCH 26/93] see if this runs --- .github/workflows/lambda_workflows/main.yml | 89 +++++++++++++++++++ .../workflows/push_docker_image_to_ecr.yml | 89 ------------------- 2 files changed, 89 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/lambda_workflows/main.yml delete mode 100644 .github/workflows/push_docker_image_to_ecr.yml diff --git a/.github/workflows/lambda_workflows/main.yml b/.github/workflows/lambda_workflows/main.yml new file mode 100644 index 0000000..4d2a9ec --- /dev/null +++ b/.github/workflows/lambda_workflows/main.yml @@ -0,0 +1,89 @@ +name: Script to deploy lambdas to aws + +on: + push: + branches: [feature/seperate_terraform_with_different_states, main] + +env: + AWS_REGION: eu-west-2 + +jobs: + build-and-push-to-ecr-for-lambda-example: + runs-on: ubuntu-latest + env: + ECR_REPOSITORY: lambda_example + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # as of 14/07/2025 it'll be using user:Junte's keys + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + # - name: Build, tag, and push Docker image to ECR + # env: + # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # IMAGE_TAG: latest + # run: | + # IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + # echo "pwd" + # pwd + # ls -la + # echo "Building Docker image..." + # docker build -t $IMAGE_URI -f deployment/lambda/lambda_example/docker/Dockerfile . + + # echo "Pushing Docker image to ECR..." + # docker push $IMAGE_URI + + # build-and-push-to-ecr-for-extractor-and-loader-example: + # runs-on: ubuntu-latest + # env: + # ECR_REPOSITORY: extractor_and_loader + + # permissions: + # id-token: write + # contents: read + + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: AWS credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # # as of 14/07/2025 it'll be using user:Junte's keys + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} + + # - name: Log in to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v2 + + # - name: Build, tag, and push Docker image to ECR + # env: + # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # IMAGE_TAG: latest + # run: | + # IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + # echo "pwd" + # pwd + # ls -la + # echo "Building Docker image..." + # docker build -t $IMAGE_URI -f deployment/lambda/extractor_and_loader/docker/Dockerfile . + + # echo "Pushing Docker image to ECR..." + # docker push $IMAGE_URI \ No newline at end of file diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml deleted file mode 100644 index d96507f..0000000 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Build and Push Docker Image to ECR lambda example - -on: - push: - branches: [feature/seperate_terraform_with_different_states, main] - -env: - AWS_REGION: eu-west-2 - -jobs: - build-and-push-to-ecr-for-lambda-example: - runs-on: ubuntu-latest - env: - ECR_REPOSITORY: lambda_example - - permissions: - id-token: write - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # as of 14/07/2025 it'll be using user:Junte's keys - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Log in to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push Docker image to ECR - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: latest - run: | - IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - echo "pwd" - pwd - ls -la - echo "Building Docker image..." - docker build -t $IMAGE_URI -f deployment/lambda/lambda_example/docker/Dockerfile . - - echo "Pushing Docker image to ECR..." - docker push $IMAGE_URI - - build-and-push-to-ecr-for-extractor-and-loader-example: - runs-on: ubuntu-latest - env: - ECR_REPOSITORY: extractor_and_loader - - permissions: - id-token: write - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # as of 14/07/2025 it'll be using user:Junte's keys - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Log in to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push Docker image to ECR - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: latest - run: | - IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - echo "pwd" - pwd - ls -la - echo "Building Docker image..." - docker build -t $IMAGE_URI -f deployment/lambda/extractor_and_loader/docker/Dockerfile . - - echo "Pushing Docker image to ECR..." - docker push $IMAGE_URI \ No newline at end of file From b6d985fb82ce14c4d4339a27b9c0712a11b0f4e7 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:24:57 +0000 Subject: [PATCH 27/93] see if lmabda runs --- .github/workflows/{lambda_workflows/main.yml => lambda_main.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{lambda_workflows/main.yml => lambda_main.yml} (100%) diff --git a/.github/workflows/lambda_workflows/main.yml b/.github/workflows/lambda_main.yml similarity index 100% rename from .github/workflows/lambda_workflows/main.yml rename to .github/workflows/lambda_main.yml From 29c1deca7c26ceb510c2b183e75228ed63c71ae0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:30:49 +0000 Subject: [PATCH 28/93] see if actions works --- .github/workflows/actions/lambda/action.yml | 7 +++++++ .github/workflows/lambda_main.yml | 22 +++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/actions/lambda/action.yml diff --git a/.github/workflows/actions/lambda/action.yml b/.github/workflows/actions/lambda/action.yml new file mode 100644 index 0000000..7c13391 --- /dev/null +++ b/.github/workflows/actions/lambda/action.yml @@ -0,0 +1,7 @@ +name: "Lambda Shared Action" +description: "Common setup for Lambda jobs" +runs: + using: "composite" + steps: + - run: echo "This is the lambda shared action" + shell: bash \ No newline at end of file diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 4d2a9ec..e9fc185 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -20,18 +20,20 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Run lambda shared action + uses: ./.github/workflows/actions/lambda - - name: AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # as of 14/07/2025 it'll be using user:Junte's keys - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + # - name: AWS credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # # as of 14/07/2025 it'll be using user:Junte's keys + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} - - name: Log in to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + # - name: Log in to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v2 # - name: Build, tag, and push Docker image to ECR # env: From 9204074eb80756466e3a7e5995646cfbbc408863 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:41:39 +0000 Subject: [PATCH 29/93] script to make terraform deploy --- .github/workflows/actions/lambda/action.yml | 41 +++++++- .../actions/terraform-deploy/action.yml | 30 ++++++ .github/workflows/lambda_main.yml | 94 ++++--------------- 3 files changed, 86 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/actions/terraform-deploy/action.yml diff --git a/.github/workflows/actions/lambda/action.yml b/.github/workflows/actions/lambda/action.yml index 7c13391..b3364de 100644 --- a/.github/workflows/actions/lambda/action.yml +++ b/.github/workflows/actions/lambda/action.yml @@ -1,7 +1,40 @@ -name: "Lambda Shared Action" -description: "Common setup for Lambda jobs" +name: "Build and Push Lambda Image to ECR" +description: "Reusable action for building and pushing lambda Docker image to ECR" + +inputs: + lambda_name: + description: "Lambda name / ECR repo name" + required: true + dockerfile_path: + description: "Path to Dockerfile" + required: true + aws_region: + description: "AWS region" + required: false + default: "eu-west-2" + runs: using: "composite" steps: - - run: echo "This is the lambda shared action" - shell: bash \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.aws_region }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and push Docker image + shell: bash + run: | + IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.lambda_name }}:latest + echo "Building Docker image for ${{ inputs.lambda_name }}..." + docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} . + + echo "Pushing to ECR..." + docker push $IMAGE_URI diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml new file mode 100644 index 0000000..d500fd6 --- /dev/null +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -0,0 +1,30 @@ +name: "Terraform Plan Shared Config" +description: "Plans shared Terraform config for Lambdas" + +inputs: + working_directory: + description: "Directory containing Terraform config" + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: ${{ inputs.working_directory }} + shell: bash + run: terraform init -reconfigure + + - name: Terraform Plan + working-directory: ${{ inputs.working_directory }} + shell: bash + run: terraform plan -out=tfplan + + - name: Terraform Apply + working-directory: ${{ inputs.working_directory }} + shell: bash + run: terraform apply -auto-approve tfplan diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index e9fc185..02fa1c6 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -1,91 +1,35 @@ -name: Script to deploy lambdas to aws +name: Lambda Main Workflow on: push: - branches: [feature/seperate_terraform_with_different_states, main] + branches: [main, feature/seperate_terraform_with_different_states] env: AWS_REGION: eu-west-2 jobs: - build-and-push-to-ecr-for-lambda-example: + lambda-ecr-example: runs-on: ubuntu-latest - env: - ECR_REPOSITORY: lambda_example - permissions: id-token: write contents: read steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run lambda shared action - uses: ./.github/workflows/actions/lambda + - name: Build and deploy Lambda example + uses: ./.github/actions/lambda-deploy + with: + lambda_name: lambda_example + dockerfile_path: deployment/lambda/lambda_example/docker/Dockerfile - # - name: AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # # as of 14/07/2025 it'll be using user:Junte's keys - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.AWS_REGION }} + extractor-and-loader: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read - # - name: Log in to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v2 - - # - name: Build, tag, and push Docker image to ECR - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # IMAGE_TAG: latest - # run: | - # IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - # echo "pwd" - # pwd - # ls -la - # echo "Building Docker image..." - # docker build -t $IMAGE_URI -f deployment/lambda/lambda_example/docker/Dockerfile . - - # echo "Pushing Docker image to ECR..." - # docker push $IMAGE_URI - - # build-and-push-to-ecr-for-extractor-and-loader-example: - # runs-on: ubuntu-latest - # env: - # ECR_REPOSITORY: extractor_and_loader - - # permissions: - # id-token: write - # contents: read - - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - - # - name: AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # # as of 14/07/2025 it'll be using user:Junte's keys - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.AWS_REGION }} - - # - name: Log in to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v2 - - # - name: Build, tag, and push Docker image to ECR - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # IMAGE_TAG: latest - # run: | - # IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - # echo "pwd" - # pwd - # ls -la - # echo "Building Docker image..." - # docker build -t $IMAGE_URI -f deployment/lambda/extractor_and_loader/docker/Dockerfile . - - # echo "Pushing Docker image to ECR..." - # docker push $IMAGE_URI \ No newline at end of file + steps: + - name: Build and deploy Extractor & Loader Lambda + uses: ./.github/actions/lambda-deploy + with: + lambda_name: extractor_and_loader + dockerfile_path: deployment/lambda/extractor_and_loader/docker/Dockerfile From efebfb5baa1b70b1ce49c18f37436607eec751b5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:51:52 +0000 Subject: [PATCH 30/93] added deployment of ecr first --- .github/workflows/actions/lambda/action.yml | 8 ++++++++ .github/workflows/lambda_main.yml | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/actions/lambda/action.yml b/.github/workflows/actions/lambda/action.yml index b3364de..0105000 100644 --- a/.github/workflows/actions/lambda/action.yml +++ b/.github/workflows/actions/lambda/action.yml @@ -8,6 +8,9 @@ inputs: dockerfile_path: description: "Path to Dockerfile" required: true + ecr_tf_dir: + description: "Path to ECR terrform directory" + required: true aws_region: description: "AWS region" required: false @@ -29,6 +32,11 @@ runs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + - name: Deploy ECR + uses: ./.github/actions/terraform-deploy + with: + working_directory: ${{ inputs.ecr_tf_dir }} + - name: Build and push Docker image shell: bash run: | diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 02fa1c6..211f8d5 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -8,8 +8,17 @@ env: AWS_REGION: eu-west-2 jobs: + shared-lambda-terraform: + runs-on: ubuntu-latest + steps: + - name: Deploy shared Lambda Config Terraform + uses: ./.github/actions/terraform-deploy + with: + working_directory: deployment/lambda_shared + lambda-ecr-example: runs-on: ubuntu-latest + needs: shared-lambda-terraform permissions: id-token: write contents: read @@ -20,9 +29,11 @@ jobs: with: lambda_name: lambda_example dockerfile_path: deployment/lambda/lambda_example/docker/Dockerfile + ecr_tf_dir: deployment/lambda/lambda_example/docker/ extractor-and-loader: runs-on: ubuntu-latest + needs: shared-lambda-terraform permissions: id-token: write contents: read @@ -33,3 +44,4 @@ jobs: with: lambda_name: extractor_and_loader dockerfile_path: deployment/lambda/extractor_and_loader/docker/Dockerfile + ecr_tf_dir: deployment/lambda/extractor_and_loader/docker/ From 7ad48ab82df2bc3a67bb741b4d68a8dce9bf1163 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:56:40 +0000 Subject: [PATCH 31/93] lambda deploy --- .github/workflows/actions/{lambda => lambda-deploy}/action.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/actions/{lambda => lambda-deploy}/action.yml (100%) diff --git a/.github/workflows/actions/lambda/action.yml b/.github/workflows/actions/lambda-deploy/action.yml similarity index 100% rename from .github/workflows/actions/lambda/action.yml rename to .github/workflows/actions/lambda-deploy/action.yml From f4ba96fba84fed915735c918168dba1098c925cd Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 10:59:26 +0000 Subject: [PATCH 32/93] check out --- .github/workflows/lambda_main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 211f8d5..fdea659 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -11,6 +11,8 @@ jobs: shared-lambda-terraform: runs-on: ubuntu-latest steps: + - name: Checkout repo + uses: actions/checkout@v4 - name: Deploy shared Lambda Config Terraform uses: ./.github/actions/terraform-deploy with: @@ -24,6 +26,8 @@ jobs: contents: read steps: + - name: Checkout repo + uses: actions/checkout@v4 - name: Build and deploy Lambda example uses: ./.github/actions/lambda-deploy with: @@ -39,6 +43,8 @@ jobs: contents: read steps: + - name: Checkout repo + uses: actions/checkout@v4 - name: Build and deploy Extractor & Loader Lambda uses: ./.github/actions/lambda-deploy with: From 52012c0f24bad1dfb5b0fe3a1acc4053e88ddef5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 12:34:14 +0000 Subject: [PATCH 33/93] check out --- .github/workflows/lambda_main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index fdea659..f069703 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -14,7 +14,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - name: Deploy shared Lambda Config Terraform - uses: ./.github/actions/terraform-deploy + uses: ./.github/workflows/actions/terraform-deploy with: working_directory: deployment/lambda_shared @@ -29,7 +29,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - name: Build and deploy Lambda example - uses: ./.github/actions/lambda-deploy + uses: ./.github/workflows/actions/lambda-deploy with: lambda_name: lambda_example dockerfile_path: deployment/lambda/lambda_example/docker/Dockerfile @@ -46,7 +46,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - name: Build and deploy Extractor & Loader Lambda - uses: ./.github/actions/lambda-deploy + uses: ./.github/workflows/actions/lambda-deploy with: lambda_name: extractor_and_loader dockerfile_path: deployment/lambda/extractor_and_loader/docker/Dockerfile From 7811f85a1772f33ea5417923c2f783295da8b673 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 12:35:27 +0000 Subject: [PATCH 34/93] directory --- .github/workflows/lambda_main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index f069703..4358ef8 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -16,7 +16,7 @@ jobs: - name: Deploy shared Lambda Config Terraform uses: ./.github/workflows/actions/terraform-deploy with: - working_directory: deployment/lambda_shared + working_directory: ./deployment/lambda_shared lambda-ecr-example: runs-on: ubuntu-latest @@ -32,8 +32,8 @@ jobs: uses: ./.github/workflows/actions/lambda-deploy with: lambda_name: lambda_example - dockerfile_path: deployment/lambda/lambda_example/docker/Dockerfile - ecr_tf_dir: deployment/lambda/lambda_example/docker/ + dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile + ecr_tf_dir: ./deployment/lambda/lambda_example/docker/ extractor-and-loader: runs-on: ubuntu-latest @@ -49,5 +49,5 @@ jobs: uses: ./.github/workflows/actions/lambda-deploy with: lambda_name: extractor_and_loader - dockerfile_path: deployment/lambda/extractor_and_loader/docker/Dockerfile - ecr_tf_dir: deployment/lambda/extractor_and_loader/docker/ + dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile + ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/ From c883f417f4f1fdf90fd7e6fccfd49a7d887d04df Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 12:37:30 +0000 Subject: [PATCH 35/93] directory --- .github/workflows/lambda_main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 4358ef8..2633600 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -16,7 +16,7 @@ jobs: - name: Deploy shared Lambda Config Terraform uses: ./.github/workflows/actions/terraform-deploy with: - working_directory: ./deployment/lambda_shared + working_directory: ./deployment/lambda/lambda_shared lambda-ecr-example: runs-on: ubuntu-latest From ee62c1920d7ee5fbc6cfc0092a67764af04304f4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 12:56:02 +0000 Subject: [PATCH 36/93] terraform mirrors deployment and aws secrets in github --- .devcontainer/Dockerfile | 6 ++++++ .devcontainer/devcontainer.json | 5 ++--- deployment/database/provider.tf | 1 - deployment/lambda/extractor_and_loader/docker/provider.tf | 1 - deployment/lambda/extractor_and_loader/provider.tf | 1 - deployment/lambda/lambda_example/docker/provider.tf | 1 - deployment/lambda/lambda_example/provider.tf | 1 - deployment/lambda/lambda_shared/provider.tf | 1 - 8 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fbf7e21..927b1ca 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,6 +3,12 @@ FROM library/python:3.12-bullseye ARG USER=vscode ARG DEBIAN_FRONTEND=noninteractive +# DO NOT PUSH IMAGE TO ECR!!! as anyone with access to image can log on to our aws +# Will log on as aws Jun-te account, change in the future to development account +ENV AWS_ACCESS_KEY_ID=AKIAU5A36PPNK7RXX52V +ENV AWS_SECRET_ACCESS_KEY=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 +ENV AWS_DEFAULT_REGION=eu-west-2 + # Install system dependencies in a single layer RUN apt update && apt install -y --no-install-recommends \ sudo jq vim curl\ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d050974..dac0087 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,9 +6,8 @@ "workspaceFolder": "/workspaces/survey-extractor", "postStartCommand": "bash .devcontainer/post-install.sh", "mounts": [ - "source=${localEnv:HOME},target=/workspaces/home,type=bind", - // Make sure you aws credentials are saved at ~/.aws - "source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind" + // Optional, just makes getting from Downloads (local env) easier + "source=${localEnv:HOME},target=/workspaces/home,type=bind" ], "customizations": { "vscode": { diff --git a/deployment/database/provider.tf b/deployment/database/provider.tf index 009e591..8f8274a 100644 --- a/deployment/database/provider.tf +++ b/deployment/database/provider.tf @@ -8,7 +8,6 @@ terraform { backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" - profile = "domna.dev" # /home/vscode/aws/credentials key = "env:/dev/terraform.tfstate" } diff --git a/deployment/lambda/extractor_and_loader/docker/provider.tf b/deployment/lambda/extractor_and_loader/docker/provider.tf index d8a8f1f..2d361d4 100644 --- a/deployment/lambda/extractor_and_loader/docker/provider.tf +++ b/deployment/lambda/extractor_and_loader/docker/provider.tf @@ -8,7 +8,6 @@ terraform { backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" - profile = "domna.dev" # /home/vscode/aws/credentials key = "env:/dev/lambda/ecr/extractor_and_loader.tfstate" } diff --git a/deployment/lambda/extractor_and_loader/provider.tf b/deployment/lambda/extractor_and_loader/provider.tf index c5008c4..51eca0c 100644 --- a/deployment/lambda/extractor_and_loader/provider.tf +++ b/deployment/lambda/extractor_and_loader/provider.tf @@ -8,7 +8,6 @@ terraform { backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" - profile = "domna.dev" # /home/vscode/aws/credentials key = "env:/dev/lambda/eachlambda/extractor_and_loader_lambda.tfstate" } diff --git a/deployment/lambda/lambda_example/docker/provider.tf b/deployment/lambda/lambda_example/docker/provider.tf index fca37f7..f210110 100644 --- a/deployment/lambda/lambda_example/docker/provider.tf +++ b/deployment/lambda/lambda_example/docker/provider.tf @@ -8,7 +8,6 @@ terraform { backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" - profile = "domna.dev" # /home/vscode/aws/credentials key = "env:/dev/lambda/ecr/lambda_example_ecr.tfstate" } diff --git a/deployment/lambda/lambda_example/provider.tf b/deployment/lambda/lambda_example/provider.tf index bac0dad..7aa8df6 100644 --- a/deployment/lambda/lambda_example/provider.tf +++ b/deployment/lambda/lambda_example/provider.tf @@ -8,7 +8,6 @@ terraform { backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" - profile = "domna.dev" # /home/vscode/aws/credentials key = "env:/dev/lambda/eachlambda/lambda_example.tfstate" } diff --git a/deployment/lambda/lambda_shared/provider.tf b/deployment/lambda/lambda_shared/provider.tf index fa6a5d4..521355b 100644 --- a/deployment/lambda/lambda_shared/provider.tf +++ b/deployment/lambda/lambda_shared/provider.tf @@ -8,7 +8,6 @@ terraform { backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" - profile = "domna.dev" # /home/vscode/aws/credentials key = "env:/dev/lambda/lambda_share_configuration.tfstate" } From 6790be41f16930c2a01efa3009c285b274e3709d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 12:59:56 +0000 Subject: [PATCH 37/93] terraform mirrors deployment and aws secrets in github --- .github/workflows/lambda_main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 2633600..fdecc83 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -51,3 +51,4 @@ jobs: lambda_name: extractor_and_loader dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/ + From f61b1ac251860ae2271c531e3b6e3987731c27d1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:10:39 +0000 Subject: [PATCH 38/93] add credentials --- .../workflows/actions/terraform-deploy/action.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml index d500fd6..ce73ffc 100644 --- a/.github/workflows/actions/terraform-deploy/action.yml +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -17,14 +17,26 @@ runs: - name: Terraform Init working-directory: ${{ inputs.working_directory }} shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} run: terraform init -reconfigure - name: Terraform Plan working-directory: ${{ inputs.working_directory }} shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} run: terraform plan -out=tfplan - name: Terraform Apply working-directory: ${{ inputs.working_directory }} shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} run: terraform apply -auto-approve tfplan From c734eabf017e29acc64334418bbc699eb97b8d03 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:17:16 +0000 Subject: [PATCH 39/93] add credentials --- .../actions/lambda-deploy/action.yml | 13 +++++++-- .../actions/terraform-deploy/action.yml | 28 +++++++++++++------ .github/workflows/lambda_main.yml | 9 ++++++ 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index 0105000..0ca72ea 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -11,6 +11,12 @@ inputs: ecr_tf_dir: description: "Path to ECR terrform directory" required: true + aws_access_key_id: + description: "AWS access key" + required: true + aws_secret_access_key: + description: "AWS secret key" + required: true aws_region: description: "AWS region" required: false @@ -24,8 +30,8 @@ runs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ inputs.aws_access_key_id }} + aws-secret-access-key: ${{ inputs.aws_secret_access_key }} aws-region: ${{ inputs.aws_region }} - name: Log in to Amazon ECR @@ -36,6 +42,9 @@ runs: uses: ./.github/actions/terraform-deploy with: working_directory: ${{ inputs.ecr_tf_dir }} + aws-access-key-id: ${{ inputs.aws_access_key_id }} + aws-secret-access-key: ${{ inputs.aws_secret_access_key }} + aws-region: ${{ inputs.aws_region }} - name: Build and push Docker image shell: bash diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml index ce73ffc..ecd60f3 100644 --- a/.github/workflows/actions/terraform-deploy/action.yml +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -5,6 +5,16 @@ inputs: working_directory: description: "Directory containing Terraform config" required: true + aws_access_key_id: + description: "AWS access key" + required: true + aws_secret_access_key: + description: "AWS secret key" + required: true + aws_region: + description: "AWS region" + required: false + default: "eu-west-2" runs: using: "composite" @@ -18,25 +28,25 @@ runs: working-directory: ${{ inputs.working_directory }} shell: bash env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} + AWS_REGION: ${{ inputs.aws_region }} run: terraform init -reconfigure - name: Terraform Plan working-directory: ${{ inputs.working_directory }} shell: bash env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} + AWS_REGION: ${{ inputs.aws_region }} run: terraform plan -out=tfplan - name: Terraform Apply working-directory: ${{ inputs.working_directory }} shell: bash env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} + AWS_REGION: ${{ inputs.aws_region }} run: terraform apply -auto-approve tfplan diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index fdecc83..abee11c 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -17,6 +17,9 @@ jobs: uses: ./.github/workflows/actions/terraform-deploy with: working_directory: ./deployment/lambda/lambda_shared + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ secrets.AWS_REGION }} lambda-ecr-example: runs-on: ubuntu-latest @@ -34,6 +37,9 @@ jobs: lambda_name: lambda_example dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/lambda_example/docker/ + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ secrets.AWS_REGION }} extractor-and-loader: runs-on: ubuntu-latest @@ -51,4 +57,7 @@ jobs: lambda_name: extractor_and_loader dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/ + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ secrets.AWS_REGION }} From 1e263ead5648f5fc5b3503d6d86173975719123f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:20:21 +0000 Subject: [PATCH 40/93] add credentials --- .github/workflows/actions/lambda-deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index 0ca72ea..0478264 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -39,7 +39,7 @@ runs: uses: aws-actions/amazon-ecr-login@v2 - name: Deploy ECR - uses: ./.github/actions/terraform-deploy + uses: ./.github/actions/workflows/terraform-deploy with: working_directory: ${{ inputs.ecr_tf_dir }} aws-access-key-id: ${{ inputs.aws_access_key_id }} From 032ca0a616ebfcf9670bdeb95a20eced7aed2ea2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:22:23 +0000 Subject: [PATCH 41/93] add credentials --- .github/workflows/actions/lambda-deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index 0478264..47f6a93 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -39,7 +39,7 @@ runs: uses: aws-actions/amazon-ecr-login@v2 - name: Deploy ECR - uses: ./.github/actions/workflows/terraform-deploy + uses: ./.github/workflows/actions/terraform-deploy with: working_directory: ${{ inputs.ecr_tf_dir }} aws-access-key-id: ${{ inputs.aws_access_key_id }} From 593b416924bd1f7e32378f8548c1645844084d5c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:31:28 +0000 Subject: [PATCH 42/93] insert aws actions --- .../actions/lambda-deploy/action.yml | 3 +-- .../actions/terraform-deploy/action.yml | 22 +++++++------------ 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index 47f6a93..f5422b5 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -19,8 +19,7 @@ inputs: required: true aws_region: description: "AWS region" - required: false - default: "eu-west-2" + required: true runs: using: "composite" diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml index ecd60f3..30d2a64 100644 --- a/.github/workflows/actions/terraform-deploy/action.yml +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -13,40 +13,34 @@ inputs: required: true aws_region: description: "AWS region" - required: false - default: "eu-west-2" + required: true runs: using: "composite" steps: - uses: actions/checkout@v4 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.aws_access_key_id }} + aws-secret-access-key: ${{ inputs.aws_secret_access_key }} + aws-region: ${{ inputs.aws_region }} + - name: Setup Terraform uses: hashicorp/setup-terraform@v3 - name: Terraform Init working-directory: ${{ inputs.working_directory }} shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} - AWS_REGION: ${{ inputs.aws_region }} run: terraform init -reconfigure - name: Terraform Plan working-directory: ${{ inputs.working_directory }} shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} - AWS_REGION: ${{ inputs.aws_region }} run: terraform plan -out=tfplan - name: Terraform Apply working-directory: ${{ inputs.working_directory }} shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} - AWS_REGION: ${{ inputs.aws_region }} run: terraform apply -auto-approve tfplan From 95b4d87d2bf98c6c478afee27c4d0067bd5605e5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:36:03 +0000 Subject: [PATCH 43/93] insert aws actions --- .../workflows/actions/lambda-deploy/action.yml | 18 +++++++++--------- .../actions/terraform-deploy/action.yml | 12 ++++++------ .github/workflows/lambda_main.yml | 12 ++++++------ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index f5422b5..e918ac1 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -11,13 +11,13 @@ inputs: ecr_tf_dir: description: "Path to ECR terrform directory" required: true - aws_access_key_id: + aws-access-key-id: description: "AWS access key" required: true - aws_secret_access_key: + aws-secret-access-key: description: "AWS secret key" required: true - aws_region: + aws-region: description: "AWS region" required: true @@ -29,9 +29,9 @@ runs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ inputs.aws_access_key_id }} - aws-secret-access-key: ${{ inputs.aws_secret_access_key }} - aws-region: ${{ inputs.aws_region }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} - name: Log in to Amazon ECR id: login-ecr @@ -41,9 +41,9 @@ runs: uses: ./.github/workflows/actions/terraform-deploy with: working_directory: ${{ inputs.ecr_tf_dir }} - aws-access-key-id: ${{ inputs.aws_access_key_id }} - aws-secret-access-key: ${{ inputs.aws_secret_access_key }} - aws-region: ${{ inputs.aws_region }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} - name: Build and push Docker image shell: bash diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml index 30d2a64..1f3ded5 100644 --- a/.github/workflows/actions/terraform-deploy/action.yml +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -5,13 +5,13 @@ inputs: working_directory: description: "Directory containing Terraform config" required: true - aws_access_key_id: + aws-access-key-id: description: "AWS access key" required: true - aws_secret_access_key: + aws-secret-access-key: description: "AWS secret key" required: true - aws_region: + aws-region: description: "AWS region" required: true @@ -23,9 +23,9 @@ runs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ inputs.aws_access_key_id }} - aws-secret-access-key: ${{ inputs.aws_secret_access_key }} - aws-region: ${{ inputs.aws_region }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} - name: Setup Terraform uses: hashicorp/setup-terraform@v3 diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index abee11c..ea18b94 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -37,9 +37,9 @@ jobs: lambda_name: lambda_example dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/lambda_example/docker/ - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: ${{ secrets.AWS_REGION }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} extractor-and-loader: runs-on: ubuntu-latest @@ -57,7 +57,7 @@ jobs: lambda_name: extractor_and_loader dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/ - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: ${{ secrets.AWS_REGION }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} From d3aa4755007ad00ebd5e1e0da7c3b3b65ede4d83 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:37:46 +0000 Subject: [PATCH 44/93] difference between underscore and hypen --- .github/workflows/lambda_main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index ea18b94..ab50fcb 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -17,9 +17,9 @@ jobs: uses: ./.github/workflows/actions/terraform-deploy with: working_directory: ./deployment/lambda/lambda_shared - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: ${{ secrets.AWS_REGION }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} lambda-ecr-example: runs-on: ubuntu-latest From 1703c9c7fd47c7833022bc0b58cb4e954630f669 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:40:37 +0000 Subject: [PATCH 45/93] difference between underscore and hypen --- .github/workflows/lambda_main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index ab50fcb..0079c1d 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -38,7 +38,7 @@ jobs: dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/lambda_example/docker/ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} extractor-and-loader: From db2d8e591f1c4a0db15b8cb1740d24762f56dab5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:45:25 +0000 Subject: [PATCH 46/93] difference between underscore and hypen --- .github/workflows/actions/lambda-deploy/action.yml | 14 ++++++++++++++ .github/workflows/lambda_main.yml | 2 ++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index e918ac1..7e8d2c2 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -11,6 +11,9 @@ inputs: ecr_tf_dir: description: "Path to ECR terrform directory" required: true + lambda_tf_dir: + description: "Path to Lambda terrform directory" + required: true aws-access-key-id: description: "AWS access key" required: true @@ -54,3 +57,14 @@ runs: echo "Pushing to ECR..." docker push $IMAGE_URI + + - name: Deploy Lambda + shell: bash + uses: ./.github/workflows/actions/terraform-deploy + with: + working_directory: ${{ inputs.lambda_tf_dir }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 0079c1d..c9cf6a8 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -37,6 +37,7 @@ jobs: lambda_name: lambda_example dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/lambda_example/docker/ + lambda_tf_dir: ./deployment/lambda/lambda_example/ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} @@ -57,6 +58,7 @@ jobs: lambda_name: extractor_and_loader dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/ + lambda_tf_dir: ./deployment/lambda/extractor_and_loader/ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} From 2ebd8b8ff3b560d6b05d3edc74a87f6450ccbd87 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 14:49:18 +0000 Subject: [PATCH 47/93] difference between underscore and hypen --- .github/workflows/actions/lambda-deploy/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index 7e8d2c2..c755e79 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -9,10 +9,10 @@ inputs: description: "Path to Dockerfile" required: true ecr_tf_dir: - description: "Path to ECR terrform directory" + description: "Path to ECR terraform directory" required: true lambda_tf_dir: - description: "Path to Lambda terrform directory" + description: "Path to Lambda terraform directory" required: true aws-access-key-id: description: "AWS access key" @@ -59,7 +59,6 @@ runs: docker push $IMAGE_URI - name: Deploy Lambda - shell: bash uses: ./.github/workflows/actions/terraform-deploy with: working_directory: ${{ inputs.lambda_tf_dir }} From f9baf95295f78ffcddc829ed90ce906ad53c0fb9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 15:07:50 +0000 Subject: [PATCH 48/93] dynamic tagging --- .../workflows/actions/lambda-deploy/action.yml | 18 +++++++++++++++++- .../actions/terraform-deploy/action.yml | 6 +++++- .github/workflows/lambda_main.yml | 4 ++++ deployment/database/variables.tf | 2 +- .../extractor_and_loader_lambda.tf | 5 +---- .../lambda_example_and_config.tf | 2 +- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index c755e79..d1328ae 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -23,6 +23,12 @@ inputs: aws-region: description: "AWS region" required: true + git-sha: + description: "Git commit SHA" + required: true + git-ref: + description: "Git ref name" + required: true runs: using: "composite" @@ -47,11 +53,19 @@ runs: aws-access-key-id: ${{ inputs.aws-access-key-id }} aws-secret-access-key: ${{ inputs.aws-secret-access-key }} aws-region: ${{ inputs.aws-region }} + - name: Set Docker image tag + id: set_tag + run: | + SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7) + BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-') + TAG="${BRANCH}-${SHORT_SHA}" + echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV + echo "tag=$TAG" >> $GITHUB_OUTPUT - name: Build and push Docker image shell: bash run: | - IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.lambda_name }}:latest + IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.lambda_name }}:${{ steps.set_tag.outputs.tag }} echo "Building Docker image for ${{ inputs.lambda_name }}..." docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} . @@ -65,5 +79,7 @@ runs: aws-access-key-id: ${{ inputs.aws-access-key-id }} aws-secret-access-key: ${{ inputs.aws-secret-access-key }} aws-region: ${{ inputs.aws-region }} + lambda-image-tag: ${{ steps.set_tag.outputs.tag }} + diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml index 1f3ded5..a978024 100644 --- a/.github/workflows/actions/terraform-deploy/action.yml +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -14,6 +14,9 @@ inputs: aws-region: description: "AWS region" required: true + lambda-image-tag: + description: "Tag of the Lambda image (e.g., GitHub SHA)" + required: false runs: using: "composite" @@ -38,7 +41,8 @@ runs: - name: Terraform Plan working-directory: ${{ inputs.working_directory }} shell: bash - run: terraform plan -out=tfplan + run: terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}" + - name: Terraform Apply working-directory: ${{ inputs.working_directory }} diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index c9cf6a8..39b9e25 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -41,6 +41,8 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} + git-sha: ${{ github.sha }} + git-ref: ${{ github.ref_name }} extractor-and-loader: runs-on: ubuntu-latest @@ -62,4 +64,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} + git-sha: ${{ github.sha }} + git-ref: ${{ github.ref_name }} diff --git a/deployment/database/variables.tf b/deployment/database/variables.tf index a237b6c..fd38c81 100644 --- a/deployment/database/variables.tf +++ b/deployment/database/variables.tf @@ -14,4 +14,4 @@ variable allocated_storage { description = "The allocated storage in gigabytes" type = number default = 20 -} \ No newline at end of file +} diff --git a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf index 91b5755..455a192 100644 --- a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf +++ b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf @@ -58,7 +58,7 @@ resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" role = data.aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:latest" + image_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:${var.lambda_image_tag}" timeout = 30 } @@ -69,6 +69,3 @@ resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" { batch_size = 1 } - -# TODO: Seperate lambda jobs from ecr creation. This is because we need to -# Create the ecr, then push images, then lambda jobs can be made \ No newline at end of file diff --git a/deployment/lambda/lambda_example/lambda_example_and_config.tf b/deployment/lambda/lambda_example/lambda_example_and_config.tf index 1fbd5ad..4f87771 100644 --- a/deployment/lambda/lambda_example/lambda_example_and_config.tf +++ b/deployment/lambda/lambda_example/lambda_example_and_config.tf @@ -57,7 +57,7 @@ resource "aws_lambda_function" "lambda_example" { function_name = "lambda-example" role = data.aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${data.aws_ecr_repository.lambda_example.repository_url}:latest" + image_uri = "${data.aws_ecr_repository.lambda_example.repository_url}:${var.lambda_image_tag}" timeout = 10 } From 582ed6ba35aacba89bbb599d47e1aaec5f852a56 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 15:10:30 +0000 Subject: [PATCH 49/93] added vars --- .github/workflows/actions/terraform-deploy/action.yml | 2 +- deployment/lambda/extractor_and_loader/vars.tf | 5 +++++ deployment/lambda/lambda_example/vars.tf | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 deployment/lambda/extractor_and_loader/vars.tf create mode 100644 deployment/lambda/lambda_example/vars.tf diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml index a978024..b2eff64 100644 --- a/.github/workflows/actions/terraform-deploy/action.yml +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -16,7 +16,7 @@ inputs: required: true lambda-image-tag: description: "Tag of the Lambda image (e.g., GitHub SHA)" - required: false + required: true runs: using: "composite" diff --git a/deployment/lambda/extractor_and_loader/vars.tf b/deployment/lambda/extractor_and_loader/vars.tf new file mode 100644 index 0000000..ecdf359 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/vars.tf @@ -0,0 +1,5 @@ +variable "lambda_image_tag" { + description = "Docker image tag (e.g. GitHub SHA)" + type = string + default = "local-dev-latest" +} \ No newline at end of file diff --git a/deployment/lambda/lambda_example/vars.tf b/deployment/lambda/lambda_example/vars.tf new file mode 100644 index 0000000..ecdf359 --- /dev/null +++ b/deployment/lambda/lambda_example/vars.tf @@ -0,0 +1,5 @@ +variable "lambda_image_tag" { + description = "Docker image tag (e.g. GitHub SHA)" + type = string + default = "local-dev-latest" +} \ No newline at end of file From fa666613e1b63dcc2e8fc197cc6cd31f0aa7f61e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 15:13:23 +0000 Subject: [PATCH 50/93] added vars --- .github/workflows/actions/terraform-deploy/action.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml index b2eff64..685a0ac 100644 --- a/.github/workflows/actions/terraform-deploy/action.yml +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -16,7 +16,7 @@ inputs: required: true lambda-image-tag: description: "Tag of the Lambda image (e.g., GitHub SHA)" - required: true + required: false runs: using: "composite" @@ -41,8 +41,12 @@ runs: - name: Terraform Plan working-directory: ${{ inputs.working_directory }} shell: bash - run: terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}" - + run: | + if [ -n "${{ inputs.lambda-image-tag }}" ]; then + terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}" + else + terraform plan -out=tfplan + fi - name: Terraform Apply working-directory: ${{ inputs.working_directory }} From 85b89a581a3ad15c4c9616204e4193cad263c007 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 15:15:16 +0000 Subject: [PATCH 51/93] include bash --- .github/workflows/actions/lambda-deploy/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml index d1328ae..ba19c67 100644 --- a/.github/workflows/actions/lambda-deploy/action.yml +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -55,6 +55,7 @@ runs: aws-region: ${{ inputs.aws-region }} - name: Set Docker image tag id: set_tag + shell: bash run: | SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7) BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-') From 792cb1d7119e305f0700278f5736c968ebaf0c34 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 15:51:07 +0000 Subject: [PATCH 52/93] add docker ignore to see if images are smaller --- deployment/lambda/extractor_and_loader/.dockerignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 deployment/lambda/extractor_and_loader/.dockerignore diff --git a/deployment/lambda/extractor_and_loader/.dockerignore b/deployment/lambda/extractor_and_loader/.dockerignore new file mode 100644 index 0000000..bd6a2d6 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/.dockerignore @@ -0,0 +1,3 @@ +*.pdf +*.xml +*.csv From b6e201c7d149f5477c3fbfc0b9200fd6a4cd8495 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:00:37 +0000 Subject: [PATCH 53/93] slimer --- .../lambda/extractor_and_loader/.dockerignore | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/deployment/lambda/extractor_and_loader/.dockerignore b/deployment/lambda/extractor_and_loader/.dockerignore index bd6a2d6..d587d34 100644 --- a/deployment/lambda/extractor_and_loader/.dockerignore +++ b/deployment/lambda/extractor_and_loader/.dockerignore @@ -1,3 +1,21 @@ +# Ignore junk and large files *.pdf -*.xml *.csv +*.xml +*.parquet +*.ipynb +*.mp4 +*.mov +*.jpg +*.png +*.zip +*.tar.gz +__pycache__/ +*.pyc +*.pyo +*.pyd +build/ +dist/ +.etl_cache/ +tests/ +docs/ From dcf589f175bcf7431fea2e250413e8c97bba9300 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:09:50 +0000 Subject: [PATCH 54/93] smaller image --- .../extractor_and_loader/docker/Dockerfile | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index cdd1f8a..878a0da 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -1,25 +1,30 @@ -FROM public.ecr.aws/lambda/python:3.12 +# ---------- Builder Stage (uses Poetry) ---------- +FROM python:3.12-slim as builder -# Install Poetry (you could pin a version if you like) -RUN curl -sSL https://install.python-poetry.org | python3 - - -# Add Poetry to PATH -ENV PATH="/root/.local/bin:$PATH" +# Install Poetry +RUN pip install --no-cache-dir poetry # Set working directory -WORKDIR /var/task +WORKDIR /app -# Copy Poetry files first to leverage Docker layer caching -COPY pyproject.toml poetry.lock README.md ./ -COPY etl/ etl/ +# Copy only what's needed for dependency resolution +COPY pyproject.toml poetry.lock ./ +# Export dependencies (no dev) to requirements.txt +RUN poetry export -f requirements.txt --without-hashes --only main -o requirements.txt -# Install dependencies into /var/task -RUN poetry config virtualenvs.create false \ - && poetry install --only main --no-interaction --no-ansi +# Install into a clean target folder +RUN pip install --no-cache-dir -r requirements.txt --target /python -# Copy app code -COPY deployment/lambda/extractor_and_loader/docker/app.py ./ +# ---------- Final Lambda Image ---------- +FROM public.ecr.aws/lambda/python:3.12 -# Set Lambda handler -CMD ["app.handler"] \ No newline at end of file +# Copy installed dependencies from builder +COPY --from=builder /python /var/task + +# Copy your app code +COPY etl/ /var/task/etl/ +COPY deployment/lambda/extractor_and_loader/docker/app.py /var/task/ + +# Set the Lambda handler (file.function_name) +CMD ["app.handler"] From f84a50a9e2a0ca1568e23fcc7cfa8e5d260b5041 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:11:58 +0000 Subject: [PATCH 55/93] don't use aws lambda --- .../extractor_and_loader/docker/Dockerfile | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index 878a0da..6fa6021 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -1,30 +1,30 @@ -# ---------- Builder Stage (uses Poetry) ---------- -FROM python:3.12-slim as builder +# Start from lightweight base image +FROM python:3.12-slim -# Install Poetry -RUN pip install --no-cache-dir poetry +# Set environment vars to reduce image size and avoid bytecode +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +# Install required system packages and Lambda RIC +RUN apt-get update && apt-get install -y curl \ + && pip install awslambdaric \ + && apt-get clean && rm -rf /var/lib/apt/lists/* # Set working directory -WORKDIR /app +WORKDIR /var/task -# Copy only what's needed for dependency resolution +# Copy Poetry files and export dependencies COPY pyproject.toml poetry.lock ./ +RUN pip install poetry \ + && poetry export -f requirements.txt --without-hashes --only main -o requirements.txt \ + && pip install -r requirements.txt --target . \ + && rm -rf ~/.cache .venv -# Export dependencies (no dev) to requirements.txt -RUN poetry export -f requirements.txt --without-hashes --only main -o requirements.txt +# Copy app code +COPY etl/ etl/ +COPY deployment/lambda/extractor_and_loader/docker/app.py . -# Install into a clean target folder -RUN pip install --no-cache-dir -r requirements.txt --target /python - -# ---------- Final Lambda Image ---------- -FROM public.ecr.aws/lambda/python:3.12 - -# Copy installed dependencies from builder -COPY --from=builder /python /var/task - -# Copy your app code -COPY etl/ /var/task/etl/ -COPY deployment/lambda/extractor_and_loader/docker/app.py /var/task/ - -# Set the Lambda handler (file.function_name) +# Lambda entrypoint +ENTRYPOINT ["/usr/local/bin/python3", "-m", "awslambdaric"] CMD ["app.handler"] From 4581b3ad82824bffea4e3a396673e7eb64033a0f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:14:41 +0000 Subject: [PATCH 56/93] don't use aws lambda --- deployment/lambda/extractor_and_loader/docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index 6fa6021..e3fde73 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -18,8 +18,7 @@ WORKDIR /var/task COPY pyproject.toml poetry.lock ./ RUN pip install poetry \ && poetry export -f requirements.txt --without-hashes --only main -o requirements.txt \ - && pip install -r requirements.txt --target . \ - && rm -rf ~/.cache .venv + && pip install -r requirements.txt --target # Copy app code COPY etl/ etl/ From 4a06e52f411691dc11d9dede4fa14b63ec80203c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:16:52 +0000 Subject: [PATCH 57/93] more minimal --- .../extractor_and_loader/docker/Dockerfile | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index e3fde73..dcbe910 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -1,29 +1,44 @@ -# Start from lightweight base image +# ----------- Build Stage (with Poetry) ----------- +FROM python:3.12-slim as builder + +ENV PIP_NO_CACHE_DIR=1 + +# Install Poetry +RUN pip install poetry + +# Set working directory +WORKDIR /app + +# Copy only what's needed to resolve dependencies +COPY pyproject.toml poetry.lock ./ + +# Export main dependencies only +RUN poetry export -f requirements.txt --without-hashes --only main -o requirements.txt + +# Install dependencies to /python (Lambda expects this path) +RUN pip install --no-deps -r requirements.txt --target /python + + +# ----------- Runtime Stage (Slim and Clean) ----------- FROM python:3.12-slim -# Set environment vars to reduce image size and avoid bytecode ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 -# Install required system packages and Lambda RIC -RUN apt-get update && apt-get install -y curl \ - && pip install awslambdaric \ - && apt-get clean && rm -rf /var/lib/apt/lists/* +# Install Lambda Runtime Interface Client (RIC) +RUN pip install awslambdaric + +# Copy app dependencies from builder +COPY --from=builder /python /var/task # Set working directory WORKDIR /var/task -# Copy Poetry files and export dependencies -COPY pyproject.toml poetry.lock ./ -RUN pip install poetry \ - && poetry export -f requirements.txt --without-hashes --only main -o requirements.txt \ - && pip install -r requirements.txt --target - -# Copy app code +# Copy application code COPY etl/ etl/ COPY deployment/lambda/extractor_and_loader/docker/app.py . # Lambda entrypoint -ENTRYPOINT ["/usr/local/bin/python3", "-m", "awslambdaric"] +ENTRYPOINT ["python3", "-m", "awslambdaric"] CMD ["app.handler"] From 39a1eb29d5cd836bc44b0b18c46bdc640cbda016 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:18:17 +0000 Subject: [PATCH 58/93] more cleaner --- .../extractor_and_loader/docker/Dockerfile | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index dcbe910..7ed1d7c 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -9,15 +9,20 @@ RUN pip install poetry # Set working directory WORKDIR /app -# Copy only what's needed to resolve dependencies +# Copy Poetry files and your source code COPY pyproject.toml poetry.lock ./ +COPY etl/ etl/ -# Export main dependencies only +# Export main dependencies RUN poetry export -f requirements.txt --without-hashes --only main -o requirements.txt -# Install dependencies to /python (Lambda expects this path) +# Install into /python (Lambda expects this structure) RUN pip install --no-deps -r requirements.txt --target /python +# Optionally: strip cache, pyc, and tests to reduce size +RUN find /python -type d -name "tests" -exec rm -rf {} + \ + && find /python -type f -name "*.pyc" -delete + # ----------- Runtime Stage (Slim and Clean) ----------- FROM python:3.12-slim @@ -26,19 +31,19 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 -# Install Lambda Runtime Interface Client (RIC) +# Install Lambda RIC RUN pip install awslambdaric -# Copy app dependencies from builder +# Copy installed dependencies and app code from builder COPY --from=builder /python /var/task +COPY --from=builder /app/etl /var/task/etl # Set working directory WORKDIR /var/task -# Copy application code -COPY etl/ etl/ +# Copy Lambda handler COPY deployment/lambda/extractor_and_loader/docker/app.py . -# Lambda entrypoint +# Set Lambda entrypoint and handler ENTRYPOINT ["python3", "-m", "awslambdaric"] CMD ["app.handler"] From 72b5b60b200e4abba36a0fd57c5d6c310e15c5a0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:22:27 +0000 Subject: [PATCH 59/93] does this build --- .../extractor_and_loader/docker/Dockerfile | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index 7ed1d7c..a314824 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -1,49 +1,41 @@ -# ----------- Build Stage (with Poetry) ----------- -FROM python:3.12-slim as builder +# ---------- Stage 1: Build (Poetry) ---------- +FROM public.ecr.aws/lambda/python:3.12 as builder -ENV PIP_NO_CACHE_DIR=1 +ENV PATH="/root/.local/bin:$PATH" \ + PIP_NO_CACHE_DIR=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 # Install Poetry -RUN pip install poetry +RUN curl -sSL https://install.python-poetry.org | python3 - -# Set working directory WORKDIR /app -# Copy Poetry files and your source code +# Copy poetry config and install deps COPY pyproject.toml poetry.lock ./ +RUN poetry config virtualenvs.create false \ + && poetry install --only main --no-interaction --no-ansi \ + && rm -rf ~/.cache + +# Copy source code COPY etl/ etl/ +COPY deployment/lambda/extractor_and_loader/docker/app.py . -# Export main dependencies -RUN poetry export -f requirements.txt --without-hashes --only main -o requirements.txt - -# Install into /python (Lambda expects this structure) -RUN pip install --no-deps -r requirements.txt --target /python - -# Optionally: strip cache, pyc, and tests to reduce size -RUN find /python -type d -name "tests" -exec rm -rf {} + \ - && find /python -type f -name "*.pyc" -delete - - -# ----------- Runtime Stage (Slim and Clean) ----------- +# ---------- Stage 2: Runtime (Slim + RIC) ---------- FROM python:3.12-slim ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 -# Install Lambda RIC +# Install AWS Lambda Runtime Interface Client RUN pip install awslambdaric -# Copy installed dependencies and app code from builder -COPY --from=builder /python /var/task -COPY --from=builder /app/etl /var/task/etl - -# Set working directory WORKDIR /var/task -# Copy Lambda handler -COPY deployment/lambda/extractor_and_loader/docker/app.py . +# Copy minimal runtime files from builder +COPY --from=builder /app /var/task -# Set Lambda entrypoint and handler +# Lambda entrypoint ENTRYPOINT ["python3", "-m", "awslambdaric"] CMD ["app.handler"] From 8dfee813c4cbb4432aa838fe92733e33fb836f4b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:25:54 +0000 Subject: [PATCH 60/93] working version --- .../extractor_and_loader/docker/Dockerfile | 52 +++++++------------ 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index a314824..cdd1f8a 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -1,41 +1,25 @@ -# ---------- Stage 1: Build (Poetry) ---------- -FROM public.ecr.aws/lambda/python:3.12 as builder +FROM public.ecr.aws/lambda/python:3.12 -ENV PATH="/root/.local/bin:$PATH" \ - PIP_NO_CACHE_DIR=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 - -# Install Poetry +# Install Poetry (you could pin a version if you like) RUN curl -sSL https://install.python-poetry.org | python3 - -WORKDIR /app - -# Copy poetry config and install deps -COPY pyproject.toml poetry.lock ./ -RUN poetry config virtualenvs.create false \ - && poetry install --only main --no-interaction --no-ansi \ - && rm -rf ~/.cache - -# Copy source code -COPY etl/ etl/ -COPY deployment/lambda/extractor_and_loader/docker/app.py . - -# ---------- Stage 2: Runtime (Slim + RIC) ---------- -FROM python:3.12-slim - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 - -# Install AWS Lambda Runtime Interface Client -RUN pip install awslambdaric +# Add Poetry to PATH +ENV PATH="/root/.local/bin:$PATH" +# Set working directory WORKDIR /var/task -# Copy minimal runtime files from builder -COPY --from=builder /app /var/task +# Copy Poetry files first to leverage Docker layer caching +COPY pyproject.toml poetry.lock README.md ./ +COPY etl/ etl/ -# Lambda entrypoint -ENTRYPOINT ["python3", "-m", "awslambdaric"] -CMD ["app.handler"] + +# Install dependencies into /var/task +RUN poetry config virtualenvs.create false \ + && poetry install --only main --no-interaction --no-ansi + +# Copy app code +COPY deployment/lambda/extractor_and_loader/docker/app.py ./ + +# Set Lambda handler +CMD ["app.handler"] \ No newline at end of file From eed6f6207bc1fd9f5c44f35716fe11951098d002 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:26:43 +0000 Subject: [PATCH 61/93] .docker ignore in wrong location --- deployment/lambda/extractor_and_loader/{ => docker}/.dockerignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename deployment/lambda/extractor_and_loader/{ => docker}/.dockerignore (100%) diff --git a/deployment/lambda/extractor_and_loader/.dockerignore b/deployment/lambda/extractor_and_loader/docker/.dockerignore similarity index 100% rename from deployment/lambda/extractor_and_loader/.dockerignore rename to deployment/lambda/extractor_and_loader/docker/.dockerignore From bf390122fab74c27fcc2e2b82b45b640f0721a81 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:31:51 +0000 Subject: [PATCH 62/93] add ecr life cycle --- .../lambda/extractor_and_loader/docker/ecr.tf | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/deployment/lambda/extractor_and_loader/docker/ecr.tf b/deployment/lambda/extractor_and_loader/docker/ecr.tf index 6b2f56a..d2c506f 100644 --- a/deployment/lambda/extractor_and_loader/docker/ecr.tf +++ b/deployment/lambda/extractor_and_loader/docker/ecr.tf @@ -24,3 +24,27 @@ resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { }) } + +# ECR lifecycle policy to delete tagged images older than 14 days +resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { + repository = aws_ecr_repository.extractor_and_loader.name + + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Delete tagged images older than 14 days" + selection = { + tagStatus = "tagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 14 + tagPrefixList = [""] # Matches all tagged images + } + action = { + type = "expire" + } + } + ] + }) +} From e316b2005b30501da041432a507d7169f2e4cea1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:34:43 +0000 Subject: [PATCH 63/93] add ecr life cycle --- deployment/lambda/extractor_and_loader/docker/ecr.tf | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/ecr.tf b/deployment/lambda/extractor_and_loader/docker/ecr.tf index d2c506f..edbe30e 100644 --- a/deployment/lambda/extractor_and_loader/docker/ecr.tf +++ b/deployment/lambda/extractor_and_loader/docker/ecr.tf @@ -25,7 +25,6 @@ resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { } -# ECR lifecycle policy to delete tagged images older than 14 days resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { repository = aws_ecr_repository.extractor_and_loader.name @@ -35,11 +34,11 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { rulePriority = 1 description = "Delete tagged images older than 14 days" selection = { - tagStatus = "tagged" - countType = "sinceImagePushed" - countUnit = "days" - countNumber = 14 - tagPrefixList = [""] # Matches all tagged images + tagStatus = "tagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 14 + # Remove tagPrefixList to apply to all tagged images } action = { type = "expire" From 60b30a49ace377a2b956a9b613bb986add900a17 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:36:24 +0000 Subject: [PATCH 64/93] add ecr life cycle --- .../lambda/extractor_and_loader/docker/ecr.tf | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/ecr.tf b/deployment/lambda/extractor_and_loader/docker/ecr.tf index edbe30e..e9f8855 100644 --- a/deployment/lambda/extractor_and_loader/docker/ecr.tf +++ b/deployment/lambda/extractor_and_loader/docker/ecr.tf @@ -25,6 +25,7 @@ resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { } +# ECR lifecycle policy to delete tagged images older than 14 days resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { repository = aws_ecr_repository.extractor_and_loader.name @@ -34,11 +35,11 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { rulePriority = 1 description = "Delete tagged images older than 14 days" selection = { - tagStatus = "tagged" - countType = "sinceImagePushed" - countUnit = "days" - countNumber = 14 - # Remove tagPrefixList to apply to all tagged images + tagStatus = "tagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 5 + tagPrefixList = [""] # Matches all tagged images } action = { type = "expire" @@ -46,4 +47,4 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { } ] }) -} +} \ No newline at end of file From a9377a3ae1798640b701114d2f84a1500bc05d2e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:37:33 +0000 Subject: [PATCH 65/93] add auto delete policy --- .../lambda/lambda_example/docker/ecr.tf | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf index d9e2eab..4e29be7 100644 --- a/deployment/lambda/lambda_example/docker/ecr.tf +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -23,3 +23,27 @@ resource "aws_ecr_repository_policy" "lambda_example_ecr_access" { }] }) } + +# ECR lifecycle policy to delete tagged images older than 14 days +resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { + repository = aws_ecr_repository.lambda_example.name + + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Delete tagged images older than 14 days" + selection = { + tagStatus = "tagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 14 + tagPrefixList = [""] # Matches all tagged images + } + action = { + type = "expire" + } + } + ] + }) +} \ No newline at end of file From a77c981d8a9420e767a95610ce84f82308e9149c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 18 Jul 2025 16:38:36 +0000 Subject: [PATCH 66/93] add auto delete policy --- deployment/lambda/extractor_and_loader/docker/ecr.tf | 9 ++++----- deployment/lambda/lambda_example/docker/ecr.tf | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/ecr.tf b/deployment/lambda/extractor_and_loader/docker/ecr.tf index e9f8855..f2b66b7 100644 --- a/deployment/lambda/extractor_and_loader/docker/ecr.tf +++ b/deployment/lambda/extractor_and_loader/docker/ecr.tf @@ -35,11 +35,10 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { rulePriority = 1 description = "Delete tagged images older than 14 days" selection = { - tagStatus = "tagged" - countType = "sinceImagePushed" - countUnit = "days" - countNumber = 5 - tagPrefixList = [""] # Matches all tagged images + tagStatus = "tagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 5 } action = { type = "expire" diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf index 4e29be7..4eb7db4 100644 --- a/deployment/lambda/lambda_example/docker/ecr.tf +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -34,11 +34,11 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { rulePriority = 1 description = "Delete tagged images older than 14 days" selection = { - tagStatus = "tagged" - countType = "sinceImagePushed" - countUnit = "days" - countNumber = 14 - tagPrefixList = [""] # Matches all tagged images + tagStatus = "tagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 5 + # Removed tagPrefixList } action = { type = "expire" From d8d359879972b4fe7f8c77981c7a5c50b70433d7 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 21 Jul 2025 09:29:58 +0000 Subject: [PATCH 67/93] terraform polciy --- .../lambda/lambda_example/docker/ecr.tf | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf index 4eb7db4..b6f1967 100644 --- a/deployment/lambda/lambda_example/docker/ecr.tf +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -29,21 +29,33 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { repository = aws_ecr_repository.lambda_example.name policy = jsonencode({ - rules = [ - { - rulePriority = 1 - description = "Delete tagged images older than 14 days" - selection = { - tagStatus = "tagged" - countType = "sinceImagePushed" - countUnit = "days" - countNumber = 5 - # Removed tagPrefixList + "rules": [ + { + "rulePriority": 1, + "description": "Expire images older than 14 days", + "selection": { + "tagStatus": "untagged", + "countType": "sinceImagePushed", + "countUnit": "days", + "countNumber": 1 + }, + "action": { + "type": "expire" + } + }, + { + "rulePriority": 1, + "description": "Keep last 5 images", + "selection": { + "tagStatus": "tagged", + "tagPrefixList": ["v"], + "countType": "imageCountMoreThan", + "countNumber": 5 + }, + "action": { + "type": "expire" + } } - action = { - type = "expire" - } - } ] }) -} \ No newline at end of file +} From c1a075bb16767a3532dd757e61d1bdfea6e774b3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 21 Jul 2025 09:33:37 +0000 Subject: [PATCH 68/93] terraform polciy only do it for feature --- deployment/lambda/lambda_example/docker/ecr.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf index b6f1967..272af42 100644 --- a/deployment/lambda/lambda_example/docker/ecr.tf +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -48,7 +48,7 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { "description": "Keep last 5 images", "selection": { "tagStatus": "tagged", - "tagPrefixList": ["v"], + "tagPrefixList": ["feature"], "countType": "imageCountMoreThan", "countNumber": 5 }, From 8fe3938116fcf54ddf8894ed6f43dc631b90de0f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 21 Jul 2025 09:35:13 +0000 Subject: [PATCH 69/93] terraform polciy only do it for feature --- deployment/lambda/lambda_example/docker/ecr.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf index 272af42..5d71101 100644 --- a/deployment/lambda/lambda_example/docker/ecr.tf +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -31,7 +31,7 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { policy = jsonencode({ "rules": [ { - "rulePriority": 1, + "rulePriority": 2, "description": "Expire images older than 14 days", "selection": { "tagStatus": "untagged", From 95aad5c78faabb0563dff02dfbe6cd2d76c9b384 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 21 Jul 2025 13:35:34 +0000 Subject: [PATCH 70/93] terraform policy --- .../lambda/extractor_and_loader/docker/ecr.tf | 39 ++++++++++++------- .../lambda/lambda_example/docker/ecr.tf | 2 +- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/ecr.tf b/deployment/lambda/extractor_and_loader/docker/ecr.tf index f2b66b7..87bedd9 100644 --- a/deployment/lambda/extractor_and_loader/docker/ecr.tf +++ b/deployment/lambda/extractor_and_loader/docker/ecr.tf @@ -30,20 +30,33 @@ resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { repository = aws_ecr_repository.extractor_and_loader.name policy = jsonencode({ - rules = [ - { - rulePriority = 1 - description = "Delete tagged images older than 14 days" - selection = { - tagStatus = "tagged" - countType = "sinceImagePushed" - countUnit = "days" - countNumber = 5 + "rules": [ + { + "rulePriority": 2, + "description": "Expire images older than 14 days", + "selection": { + "tagStatus": "untagged", + "countType": "sinceImagePushed", + "countUnit": "days", + "countNumber": 1 + }, + "action": { + "type": "expire" + } + }, + { + "rulePriority": 1, + "description": "Keep last 5 images", + "selection": { + "tagStatus": "tagged", + "tagPrefixList": ["feature"], + "countType": "imageCountMoreThan", + "countNumber": 5 + }, + "action": { + "type": "expire" + } } - action = { - type = "expire" - } - } ] }) } \ No newline at end of file diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf index 5d71101..276697f 100644 --- a/deployment/lambda/lambda_example/docker/ecr.tf +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -25,7 +25,7 @@ resource "aws_ecr_repository_policy" "lambda_example_ecr_access" { } # ECR lifecycle policy to delete tagged images older than 14 days -resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { +resource "aws_ecr_lifecycle_policy" "lambda_example_ecr_lifecycle" { repository = aws_ecr_repository.lambda_example.name policy = jsonencode({ From 1d347098b70013dff9ae2eea587f2c1c3523f50c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 22 Jul 2025 12:56:56 +0000 Subject: [PATCH 71/93] sanity month end automtaion files --- etl/month_end_automation.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 etl/month_end_automation.py diff --git a/etl/month_end_automation.py b/etl/month_end_automation.py new file mode 100644 index 0000000..e75154b --- /dev/null +++ b/etl/month_end_automation.py @@ -0,0 +1 @@ +print("hello world") \ No newline at end of file From d4d5531ddc9c7c1e29137742f5513112bc2cef46 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 22 Jul 2025 17:18:21 +0100 Subject: [PATCH 72/93] months end --- etl/month_end_automation.py | 65 +++++++++++++++++++++- etl/osmosis_complaince_address_to_files.py | 60 ++++++++++---------- 2 files changed, 94 insertions(+), 31 deletions(-) diff --git a/etl/month_end_automation.py b/etl/month_end_automation.py index e75154b..d396551 100644 --- a/etl/month_end_automation.py +++ b/etl/month_end_automation.py @@ -1 +1,64 @@ -print("hello world") \ No newline at end of file +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +board_ids = ["3900434153"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + else: + + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +# RA +filtered_df = df[ + df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) +].copy() + +# ATT +filtered_df = df[ + df["att"].str.lower().isin(["completed"]) +].copy() + +# V1 Coordination +filtered_df = df[ + df["v1 coordination status"].str.lower().isin(["rc complete"]) +].copy() + +# V2 Coordination +_ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] +filtered_df = _[_["v2 dc/ima/pas"] > 0] + diff --git a/etl/osmosis_complaince_address_to_files.py b/etl/osmosis_complaince_address_to_files.py index 0ab6ed9..c48e4eb 100644 --- a/etl/osmosis_complaince_address_to_files.py +++ b/etl/osmosis_complaince_address_to_files.py @@ -91,7 +91,7 @@ def get_all_items(board_id, monday): limit = 25 # Adjust the limit based on how many items you want per request all_items = [] # List to store all fetched items cursor = None # Start without a cursor for the first page - + print(f"Connecting to Monday API and retrieving data for board {board_id}") # Loop through pages while True: # Fetch items for the current page @@ -116,8 +116,7 @@ def get_all_items(board_id, monday): # If there's no cursor, we've reached the last page if not cursor: break - print(f"cursor {cursor}") - print(f"len all_itemms {len(all_items)}") + print("Loading...") return all_items def upload_to_sharepoint(to_upload, master_folder_name): @@ -128,36 +127,37 @@ def upload_to_sharepoint(to_upload, master_folder_name): print(f"Uploading {file_name} to sharepoint") osmosis.upload_file(file_path, parent_folder + f"/{master_folder_name}", file_name) -# Step 1: Fetch column IDs -board_data = monday.boards.fetch_boards_by_id(board_id) -columns = board_data["data"]["boards"][0]["columns"] -col_id_map = {col["title"].lower(): col["id"] for col in columns} +if __name__ == "__main__": + # Step 1: Fetch column IDs + board_data = monday.boards.fetch_boards_by_id(board_id) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} -name_id = col_id_map.get("name") # Replace with actual title if different -files_id = col_id_map.get("file(s)") # Replace with actual title if different + name_id = col_id_map.get("name") # Replace with actual title if different + files_id = col_id_map.get("file(s)") # Replace with actual title if different -if not name_id or not files_id: - raise Exception("Could not find 'name' or 'file(s)' columns") + if not name_id or not files_id: + raise Exception("Could not find 'name' or 'file(s)' columns") -items = get_all_items(board_id, monday) -for i,item in enumerate(tqdm(items)): - if i>329: - item_name = item["name"] - item_name = sanitize_name(item_name, ignore_dot=True) - print(f"Item name is {item_name}") - asset_ids = extract_asset_ids(item, files_id) + items = get_all_items(board_id, monday) + for i,item in enumerate(tqdm(items)): + if i>329: + item_name = item["name"] + item_name = sanitize_name(item_name, ignore_dot=True) + print(f"Item name is {item_name}") + asset_ids = extract_asset_ids(item, files_id) - to_upload = [] - for asset_id in asset_ids: - try: - public_url, file_name = get_public_url(asset_id) - print(f"Downloading {file_name}") - file_path = download_file_from_public_url(public_url, file_name) - to_upload.append(file_path) - except Exception as e: - print(f"Failed to download/upload asset {asset_id}: {e}") + to_upload = [] + for asset_id in asset_ids: + try: + public_url, file_name = get_public_url(asset_id) + print(f"Downloading {file_name}") + file_path = download_file_from_public_url(public_url, file_name) + to_upload.append(file_path) + except Exception as e: + print(f"Failed to download/upload asset {asset_id}: {e}") - if to_upload: - upload_to_sharepoint(to_upload, item_name) + if to_upload: + upload_to_sharepoint(to_upload, item_name) -# Liv green # Cocuun # Wates \ No newline at end of file + # Liv green # Cocuun # Wates \ No newline at end of file From 1741c647afe8c8a54093bc01956e58ce02fd4cd2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 23 Jul 2025 10:27:58 +0000 Subject: [PATCH 73/93] save month ened wave 2 one board example --- etl/month_end_automation.py | 107 ++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/etl/month_end_automation.py b/etl/month_end_automation.py index d396551..2eb9e68 100644 --- a/etl/month_end_automation.py +++ b/etl/month_end_automation.py @@ -1,3 +1,5 @@ +# Wave 2's month end automation + from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids @@ -23,6 +25,7 @@ for board in tqdm(board_ids): for row in tqdm(items): data = {} data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) for col in row.get("column_values", []): if col.get("id") in reversed_col_id_map: if col.get("type") == "file": @@ -33,8 +36,27 @@ for board in tqdm(board_ids): value = json.loads(col["value"]) no_of_files = len(value.get('files', [])) data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) else: - data.update({ reversed_col_id_map[col.get("id")]: col.get("text") }) @@ -43,22 +65,97 @@ for board in tqdm(board_ids): # Convert to DataFrame df = pd.DataFrame(all_records) +filtered_dfs = [] + # RA -filtered_df = df[ +ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) ].copy() +filtered_dfs.append(ra) + # ATT -filtered_df = df[ +att = df[ df["att"].str.lower().isin(["completed"]) ].copy() +filtered_dfs.append(att) # V1 Coordination -filtered_df = df[ +v1 = df[ df["v1 coordination status"].str.lower().isin(["rc complete"]) ].copy() +filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] -filtered_df = _[_["v2 dc/ima/pas"] > 0] +v2 = _[_["v2 dc/ima/pas"] > 0] +filtered_dfs.append(v2) + +# V3 Coordination +v3 = df[ + df["v3 invoiced"].str.lower().isin(["to be invoiced"]) +].copy() +filtered_dfs.append(v3) + +# Design stage 1 +design1 = df[ + df["design invoice status"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(design1) + +# Design stage 2 +design2 = df[ + df["revision 2 design invoice"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = df[ + df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +lodg2 = df[ + df["full lodgement invoice status"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(lodg2) + +# POST EPC +# !!! Need Kevins input ( how does he determine invoice in this one) + +# Post ATT +# !!! Check with kevin, if he uses epc pic uploaded ( to invoice ) or uses anything else +filtered_df = df[ + df["post-att"].str.lower().isin(["post-att uploaded"]) +] + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["complete"]) +] +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +] +filtered_dfs.append(ra_ns) + + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +] +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +] +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True) From e1842b4150c265f76de4a17b9f7d6899729dfad0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 23 Jul 2025 10:55:55 +0000 Subject: [PATCH 74/93] month end automation --- etl/month_end_automation.py | 56 ++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/etl/month_end_automation.py b/etl/month_end_automation.py index 2eb9e68..a3be693 100644 --- a/etl/month_end_automation.py +++ b/etl/month_end_automation.py @@ -71,6 +71,7 @@ filtered_dfs = [] ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) ].copy() +ra["job_type"] = "RA" filtered_dfs.append(ra) @@ -78,68 +79,94 @@ filtered_dfs.append(ra) att = df[ df["att"].str.lower().isin(["completed"]) ].copy() +att["job_type"] = "ATT" filtered_dfs.append(att) # V1 Coordination v1 = df[ df["v1 coordination status"].str.lower().isin(["rc complete"]) ].copy() +v1["job_type"] = "V1 Coordination" filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] -v2 = _[_["v2 dc/ima/pas"] > 0] +v2 = _[_["v2 dc/ima/pas"] > 0].copy() +v2["job_type"] = "V2 Coordination" filtered_dfs.append(v2) # V3 Coordination v3 = df[ df["v3 invoiced"].str.lower().isin(["to be invoiced"]) ].copy() +v3["job_type"] = "V3 Coordination" filtered_dfs.append(v3) # Design stage 1 design1 = df[ df["design invoice status"].str.lower().isin(["to invoice"]) -] +].copy() +design1["job_type"] = "Design" filtered_dfs.append(design1) # Design stage 2 design2 = df[ df["revision 2 design invoice"].str.lower().isin(["to invoice"]) -] +].copy() +design2["job_type"] = "Design Revision" filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = df[ df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) -] +].copy() +lodg1["job_type"] = "Lodgement Phase 1" filtered_dfs.append(lodg1) # Full Lodgement Phase lodg2 = df[ df["full lodgement invoice status"].str.lower().isin(["to invoice"]) -] +].copy() +lodg2["job_type"] = "Full Lodgement" filtered_dfs.append(lodg2) # POST EPC -# !!! Need Kevins input ( how does he determine invoice in this one) +post_epc = df[ + df["post-epc status"].str.lower().isin(["epc pics uploaded"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) + + # Post ATT -# !!! Check with kevin, if he uses epc pic uploaded ( to invoice ) or uses anything else -filtered_df = df[ +post_att = df[ df["post-att"].str.lower().isin(["post-att uploaded"]) -] +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + # Retrofit Evaluation retro = df[ df["retrofit evaluation"].str.lower().isin(["complete"]) -] +].copy() +retro["job_type"] = "Retrofit Evaluation" filtered_dfs.append(retro) # RA NO Show ra_ns = df[ df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) -] +].copy() +ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -147,15 +174,18 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) -] +].copy() +att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) -] +].copy() +epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True) +final_df[['address', 'client', 'job_type']] \ No newline at end of file From 542a788a85309bf90ce02b6858206bd9af96a270 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 23 Jul 2025 13:08:50 +0000 Subject: [PATCH 75/93] month end automation --- etl/month_end_automation.py | 3 +- etl/month_end_automation_wave_3_layout.py | 197 ++++++++++++++++++++++ 2 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 etl/month_end_automation_wave_3_layout.py diff --git a/etl/month_end_automation.py b/etl/month_end_automation.py index a3be693..69c91ba 100644 --- a/etl/month_end_automation.py +++ b/etl/month_end_automation.py @@ -9,6 +9,7 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) +# NCHK SHDF Westville Wave 1 & 2 board_ids = ["3900434153"] @@ -17,7 +18,7 @@ for board in tqdm(board_ids): columns = board_data["data"]["boards"][0]["columns"] col_id_map = {col["title"].lower(): col["id"] for col in columns} reversed_col_id_map = {v: k for k, v in col_id_map.items()} - + items = get_all_items(board, monday) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py new file mode 100644 index 0000000..2fd7ebe --- /dev/null +++ b/etl/month_end_automation_wave_3_layout.py @@ -0,0 +1,197 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# WCHG Walkups-Operations +board_ids = ["9349630181"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + # elif "no show" in reversed_col_id_map[col.get("id")]: + # def extract_number_from_text(text): + # number_str = '' + + # for char in text: + # if char.isnumeric(): + # number_str += char + # elif number_str: + # break # stop once a number sequence ends + + # return int(number_str) if number_str else None + # text = col.get("text") + # if text is None: + # data.update({ + # reversed_col_id_map[col.get("id")]: col.get("text") + # }) + # else: + # data.update({ + # reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + # }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["ra invoicing status"].str.lower().isin(["to invoice"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["pre att invoicing status"].str.lower().isin(["to invoice"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["mtp invoicing status"].str.lower().isin(["(v1) ioe/mtp to invoice"]) +].copy() + +v1["job_type"] = "V1 Coordination" +filtered_dfs.append(v1) + +# V2 Coordination +v2 = df[ + df["mtp invoicing status"].str.lower().isin(["(v2) ioe/mtp to invoice"]) +].copy() + +v2["job_type"] = "V2 Coordination" +filtered_dfs.append(v2) + +# V3 Coordination +v3 = df[ + df["mtp invoicing status"].str.lower().isin(["(v3) ioe/mtp to invoice"]) +].copy() + +v3["job_type"] = "V3 Coordination" +filtered_dfs.append(v3) + +# Design stage 1 +design1 = df[ + df["design invoicing status"].str.lower().isin(["to invoice"]) +].copy() +design1["job_type"] = "Design" +filtered_dfs.append(design1) + +# Design Revision +revision_letter = ['a', 'b', 'c', 'd'] +for letter in revision_letter: + design2 = df[ + df["design revision invoice"].str.lower().isin([f"rev. {letter} to invoice"]) + ].copy() + design2["job_type"] = "Design Revision" + filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = df[ + df["lodgement phase 1 invoicing status"].str.lower().isin(["to invoice"]) +].copy() +lodg1["job_type"] = "Lodgement Phase 1" +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +lodg2 = df[ + df["full lodgement invoicing status"].str.lower().isin(["to invoice"]) +].copy() +lodg2["job_type"] = "Full Lodgement" +filtered_dfs.append(lodg2) + +# POST EPC +post_epc = df[ + df["post epc & eval. invoicing status"].str.lower().isin(["to invoice"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# # POST EPR, not right now +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + + +# Post ATT +post_att = df[ + df["post att invoicing status"].str.lower().isin(["to invoice"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["rc stage 2 invoicing status"].str.lower().isin(["to invoice"]) +].copy() +retro["job_type"] = "RC stage 2" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["pre att no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +#post work no show invoice +epc_ns = df[ + df["post works no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file From d2e57de89441daeb1192c4010db5cde7c2abcaa1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 23 Jul 2025 13:11:21 +0000 Subject: [PATCH 76/93] wave 3 automation --- etl/month_end_automation_wave_3_layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 2fd7ebe..102f99e 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -1,4 +1,4 @@ -# Wave 2's month end automation +# Wave 3's month end automation from tqdm import tqdm from monday import MondayClient From 4ff30fd8ef1f7a39b5292f94a1fe03a9aacc5573 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 23 Jul 2025 15:55:12 +0000 Subject: [PATCH 77/93] save current work --- ... => month_end_automation_wave_2_layout.py} | 2 +- etl/month_end_automation_wave_2_no_3.py | 191 +++++++++++++++++ etl/month_end_automation_wave_2_no_4.py | 192 ++++++++++++++++++ etl/month_end_automation_wave_3_layout.py | 8 +- 4 files changed, 388 insertions(+), 5 deletions(-) rename etl/{month_end_automation.py => month_end_automation_wave_2_layout.py} (98%) create mode 100644 etl/month_end_automation_wave_2_no_3.py create mode 100644 etl/month_end_automation_wave_2_no_4.py diff --git a/etl/month_end_automation.py b/etl/month_end_automation_wave_2_layout.py similarity index 98% rename from etl/month_end_automation.py rename to etl/month_end_automation_wave_2_layout.py index 69c91ba..843e2fa 100644 --- a/etl/month_end_automation.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -187,6 +187,6 @@ epc_ns = df[ epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) -final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True) +final_df = pd.concat(filtered_dfs).reset_index(drop=True) final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py new file mode 100644 index 0000000..8a7ca4c --- /dev/null +++ b/etl/month_end_automation_wave_2_no_3.py @@ -0,0 +1,191 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Platform Housing W2 (in use) +board_ids = ["4796290860"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9", "completed", "complete"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["coordination status"].str.lower().isin(["ima/mtp completed"]) +].copy() +v1["job_type"] = "V1 Coordination" +filtered_dfs.append(v1) + +# V2 Coordination +_ = df[df["v2 mtp status"].fillna('').str.lower().isin(['v2 ima-mtp completed', 'v2 completed'])].copy() +_["job_type"] = "V2 Coordination" +filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[ +# df["v3 invoiced"].str.lower().isin(["to be invoiced"]) +# ].copy() +# v3["job_type"] = "V3 Coordination" +# filtered_dfs.append(v3) + +# Design stage 1 +design1 = df[ + df["design invoice"].str.lower().isin(["complete pending rc"]) +].copy() +design1["job_type"] = "Design" +filtered_dfs.append(design1) + +# Design stage 2 +# design2 = df[ +# df["revision 2 design invoice"].str.lower().isin(["to invoice"]) +# ].copy() +# design2["job_type"] = "Design Revision" +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = df[ + df["phase 1 invoice status (lodgement)"].str.lower().isin(["done"]) +].copy() +lodg1["job_type"] = "Lodgement Phase 1" +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +lodg2 = df[ + df["lodgement invoice status (lodgement)"].str.lower().isin(["to invoice"]) +].copy() +lodg2["job_type"] = "Full Lodgement" +filtered_dfs.append(lodg2) + +# POST EPC +post_epc = df[ + df["post epc"].str.lower().isin(["success", "pics uploaded"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + + + +# Post ATT +post_att = df[ + df["post att"].str.lower().isin(["uploaded"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py new file mode 100644 index 0000000..7a4cc41 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_4.py @@ -0,0 +1,192 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Stonewater (in use) +board_ids = ["3584401309"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["v1 coordination status (ioe,mtp)"].str.lower().isin(["rc complete"]) +].copy() +v1["job_type"] = "V1 Coordination" +filtered_dfs.append(v1) + +# V2 Coordination +_ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +_["job_type"] = "V2 Coordination" +filtered_dfs.append(_) + +# V3 Coordination +v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +v3["job_type"] = "V3 Coordination" +filtered_dfs.append(_) + +# Design stage 1 +design1 = df[ + df["design invoice status"].str.lower().isin(["complete", "to invoice"]) +].copy() +design1["job_type"] = "Design" +filtered_dfs.append(design1) + +# Design stage 2 +# design2 = df[ +# df[""].str.lower().isin(["to invoice"]) +# ].copy() +# design2["job_type"] = "Design Revision" +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = df[ + df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +].copy() +lodg1["job_type"] = "Lodgement Phase 1" +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +_ = df[ + df["lodgement status"].str.lower().isin(["lodged"]) +].copy() +lodg2 = _[ + _["lodgement invoice status"].str.lower().isin([None]) +].copy() +lodg2["job_type"] = "Full Lodgement" +filtered_dfs.append(lodg2) + +# POST EPC +post_epc = df[ + df["post epc"].str.lower().isin(["success", "pics uploaded"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + + + +# Post ATT +post_att = df[ + df["post att"].str.lower().isin(["uploaded"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 102f99e..586b6da 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -170,7 +170,7 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) + df["ra no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -178,7 +178,7 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["pre att no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) + df["pre att no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -187,11 +187,11 @@ filtered_dfs.append(att_ns) # Post visit no show #post work no show invoice epc_ns = df[ - df["post works no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) + df["post works no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) -final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True) +final_df = pd.concat(filtered_dfs).reset_index(drop=True) final_df[['address', 'client', 'job_type']] \ No newline at end of file From c6ec1eac5da5c5d51634638cede00cfd2dd0cf53 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 24 Jul 2025 09:51:15 +0000 Subject: [PATCH 78/93] push to this branch --- etl/hubSpotClient/hubspot.py | 5 +- ...pull_data_from_hubspot_and_ideally_push.py | 53 +++++++++++++++++++ etl/surveyedData/surveryedData.py | 2 + 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 3b4913e..269862c 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -206,7 +206,7 @@ class HubSpotClient(): after = response.paging.next.after all_deals = [] - for deal in found_deals: + for i,deal in enumerate(found_deals): domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id) try: deal_name = deal.properties['dealname'] @@ -226,6 +226,9 @@ class HubSpotClient(): domna_id = domna_id, uprn = uprn, )) + if i > 1: + # Break after just for speed + break except Exception as e: def format_error_note(e): note_text = "⚠️ Automated Verification Failed:

" diff --git a/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py b/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py new file mode 100644 index 0000000..34924cc --- /dev/null +++ b/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py @@ -0,0 +1,53 @@ +import os +from pprint import pprint + +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" + +from etl.hubSpotClient.hubspot import DealStage, HubSpotClient +from etl.surveyedData.surveryedData import surveyedDataProcessor +from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.utils.utils import get_sharepoint_path + +def string_to_installer(installer): + if installer.upper() == "J & J CRUMP": + return SharePointInstaller.JJC + elif installer.upper() == "SCIS": + return SharePointInstaller.SOUTH_COAST_INSULATION + elif installer.upper() == "SGEC": + return SharePointInstaller.JJC + else: + return None + +# Local development +os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" + +hubspotClient = HubSpotClient() + +# Gets all deals and puts it into a SubmissionInfoFromDeal class +# KHALIM - I ADDED A SCRIPT TO ONLY DOWNLOAD 1 deal for speed sake +deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + + +for deal in deals: + sharepoint_url = deal.submission_folder_path + installer = string_to_installer(deal.installer) + sp = SharePointScraper(installer) + path = get_sharepoint_path(sharepoint_url) + + files = sp.download_files_from_path(path) + sdp = surveyedDataProcessor("fake address", files) + + # Class Object for EPR Summary Informaiton ( Transform ) + sdp.epr_summary_information + + # File path to epr + sdp.epr_summary_information_file_path + + break + + + diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index f75b112..a88b8da 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -43,6 +43,7 @@ class surveyedDataProcessor(): self.hubspot_deal_id = None self.epr_with_data = None self.epr_summary_information = None + self.epr_summary_information_file_path = None self.full_sap_xml = None self.lig_sap_xml = None self.rd_sap_xml = None @@ -69,6 +70,7 @@ class surveyedDataProcessor(): self.epr_with_data = pdf.get_reader() elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION: self.epr_summary_information = pdf.get_reader() + self.epr_summary_information_file_path = file elif file.lower().endswith('.xml'): xml = xmlReader(file) From 683e1d792edbc1a86338809d20d06749c3e8e501 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 24 Jul 2025 10:00:30 +0000 Subject: [PATCH 79/93] save --- etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py b/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py index 34924cc..fdb15fe 100644 --- a/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py +++ b/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py @@ -38,7 +38,7 @@ for deal in deals: sp = SharePointScraper(installer) path = get_sharepoint_path(sharepoint_url) - files = sp.download_files_from_path(path) + files = sp.downloadt_files_from_path(path) sdp = surveyedDataProcessor("fake address", files) # Class Object for EPR Summary Informaiton ( Transform ) From c3b7345eabcf66944d66d46ddb980d7c70690d42 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 24 Jul 2025 14:22:32 +0000 Subject: [PATCH 80/93] completed --- etl/month_end_automation_wave_2_no_10.py | 170 +++++++++++++++++++++ etl/month_end_automation_wave_2_no_4.py | 18 +-- etl/month_end_automation_wave_2_no_5.py | 186 +++++++++++++++++++++++ etl/month_end_automation_wave_2_no_6.py | 175 +++++++++++++++++++++ etl/month_end_automation_wave_2_no_7.py | 169 ++++++++++++++++++++ etl/month_end_automation_wave_2_no_8.py | 172 +++++++++++++++++++++ etl/month_end_automation_wave_2_no_9.py | 169 ++++++++++++++++++++ 7 files changed, 1047 insertions(+), 12 deletions(-) create mode 100644 etl/month_end_automation_wave_2_no_10.py create mode 100644 etl/month_end_automation_wave_2_no_5.py create mode 100644 etl/month_end_automation_wave_2_no_6.py create mode 100644 etl/month_end_automation_wave_2_no_7.py create mode 100644 etl/month_end_automation_wave_2_no_8.py create mode 100644 etl/month_end_automation_wave_2_no_9.py diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py new file mode 100644 index 0000000..87580f2 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_10.py @@ -0,0 +1,170 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA Derbyshire Dales (DDDCC) SHDF +board_ids = ["6947307148"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "lodged epc", ["complete"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["done"], "POST ATT") +filtered_dfs.append(post_att) + +post_att = get_df(df, "post-test status", ["complete"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 7a4cc41..6fe0d90 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -123,17 +123,14 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase _ = df[ - df["lodgement status"].str.lower().isin(["lodged"]) + df["lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() -lodg2 = _[ - _["lodgement invoice status"].str.lower().isin([None]) -].copy() -lodg2["job_type"] = "Full Lodgement" -filtered_dfs.append(lodg2) +_["job_type"] = "Full Lodgement" +filtered_dfs.append(_) # POST EPC post_epc = df[ - df["post epc"].str.lower().isin(["success", "pics uploaded"]) + df["post epc"].str.lower().isin(["completed & uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) @@ -146,11 +143,9 @@ filtered_dfs.append(post_epc) # post_epr["job_type"] = "POST ATT" # filtered_dfs.append(post_epr) - - # Post ATT post_att = df[ - df["post att"].str.lower().isin(["uploaded"]) + df["post att"].str.lower().isin(["completed & uploaded"]) ].copy() post_att["job_type"] = "POST ATT" filtered_dfs.append(post_att) @@ -158,7 +153,7 @@ filtered_dfs.append(post_att) # Retrofit Evaluation retro = df[ - df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"]) + df["retrofit evaluation"].str.lower().isin(["completed & uploaded"]) ].copy() retro["job_type"] = "Retrofit Evaluation" filtered_dfs.append(retro) @@ -171,7 +166,6 @@ ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) - # ATT NO Show att_ns = df[ df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py new file mode 100644 index 0000000..daaa8b0 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_5.py @@ -0,0 +1,186 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# ECO 4 NCHA Almshouses Operations +board_ids = ["9136254638"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["retrofit assessment"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["pre att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["coordination status"].str.lower().isin(["ioe/mtp complete"]) +].copy() +v1["job_type"] = "V1 Coordination" +filtered_dfs.append(v1) + +# V2 Coordination +# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +# _["job_type"] = "V2 Coordination" +# filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +# v3["job_type"] = "V3 Coordination" +# filtered_dfs.append(_) + +# Design stage 1 +design1 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design1["job_type"] = "Design" +filtered_dfs.append(design1) + +# Design stage 2 +# design2 = df[ +# df[""].str.lower().isin(["to invoice"]) +# ].copy() +# design2["job_type"] = "Design Revision" +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = df[ +# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +# ].copy() +# lodg1["job_type"] = "Lodgement Phase 1" +# filtered_dfs.append(lodg1) + +# Full Lodgement Phase +_ = df[ + df["trustmark lodgement"].str.lower().isin(["done"]) +].copy() +_["job_type"] = "Full Lodgement" +filtered_dfs.append(_) + +# POST EPC +post_epc = df[ + df["post epc status"].str.lower().isin(["done"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = df[ + df["post att status"].str.lower().isin(["done"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["done"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py new file mode 100644 index 0000000..d069018 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_6.py @@ -0,0 +1,175 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +#ECO 4 Wates Operations +board_ids = ["9520779048"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra status", ["completed & uploaded"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre att status", ["completed & uploaded"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "Coordination Status IOE/MTP".lower(), [ + "(V1) IOE/MTP Complete".lower(), + "(V2) IOE/MTP Complete".lower(), + "(V3) IOE/MTP Complete".lower(), +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +# _["job_type"] = "V2 Coordination" +# filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +# v3["job_type"] = "V3 Coordination" +# filtered_dfs.append(_) + +# Design stage 1 +design1 = get_df(df, "retrofit design status", ["completed"], "Design") +filtered_dfs.append(design1) + +# Design stage 2 +design2 = get_df(df, "design revision invoice", [ + "Rev. A to invoice".lower(), + "Rev. B to invoice".lower(), + "Rev. C to invoice".lower(), + "Rev. D to invoice".lower(), +], "Design Revision") +filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = df[ +# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +# ].copy() +# lodg1["job_type"] = "Lodgement Phase 1" +# filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement", ["completed"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc & evaluation status", ["uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att status", ["uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +# ra_ns = df[ +# df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +# ].copy() +# ra_ns["job_type"] = "RA NO SHOW" +# filtered_dfs.append(ra_ns) + + +# # ATT NO Show +# att_ns = df[ +# df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# # Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py new file mode 100644 index 0000000..fb15205 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_7.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +#Home Group Wave 2SP+ +board_ids = ["4254419092"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre-att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "osmosis rc status".lower(), [ + "rc completed", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 ioe mtp", ["completed"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["complete & uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py new file mode 100644 index 0000000..ddd2368 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_8.py @@ -0,0 +1,172 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA SHDF 2.1 SBS +board_ids = ["8668578700"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "v1 coordination status".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Lodgement Phase 1 +lodg1 = get_df(df, "phase 1 to be invoiced".lower(), ["phase 1 to be invoiced"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py new file mode 100644 index 0000000..ca198f5 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_9.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA Almshouses +board_ids = ["5423364294"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status (mtp)".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +# design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") +# filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +# post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC") +# filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +# post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT") +# filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file From 25b706af076d26f91f7b89746ad1ad0512e2c840 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 24 Jul 2025 15:41:21 +0000 Subject: [PATCH 81/93] save existing --- etl/month_end_automation_wave_2_no_11.py | 170 +++++++++++++++++++++++ etl/month_end_automation_wave_2_no_12.py | 167 ++++++++++++++++++++++ etl/month_end_automation_wave_2_no_13.py | 166 ++++++++++++++++++++++ 3 files changed, 503 insertions(+) create mode 100644 etl/month_end_automation_wave_2_no_11.py create mode 100644 etl/month_end_automation_wave_2_no_12.py create mode 100644 etl/month_end_automation_wave_2_no_13.py diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py new file mode 100644 index 0000000..babf1f8 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_11.py @@ -0,0 +1,170 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Northumberland LAD2 & HUG2 +board_ids = ["5121300882"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "lite ima status".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "ima-mtp status", ["ima-mtp completed"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# filtered_dfs.append(v3) + +# Design stage 1 +# design1 = get_df(df, "", ["done"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "tm phase 1 invoiced".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["uploaded & completed", "to invoice"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att status", ["uploaded & completed", "to invoice"], "POST ATT") +filtered_dfs.append(post_att) + +post_att = get_df(df, "post-test status", ["complete"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done", "to invoice"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +# ra_ns = df[ +# df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +# ].copy() +# ra_ns["job_type"] = "RA NO SHOW" +# filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py new file mode 100644 index 0000000..261ed2f --- /dev/null +++ b/etl/month_end_automation_wave_2_no_12.py @@ -0,0 +1,167 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Shropshire Council HUG2 +board_ids = ["4718185486"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre- att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "ima lite invoiced".lower(), [ + "to invoice", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "coordination v2 invoiced", ["to invoice"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# filtered_dfs.append(v3) + +# Design stage 1 +# design1 = get_df(df, "", ["done"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed", "to invoice"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +# att_ns = df[ +# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py new file mode 100644 index 0000000..412c6f7 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_13.py @@ -0,0 +1,166 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Stonewater SHDF 3.0 - Operations +board_ids = ["6222522864"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status (mtp)".lower(), [ + "ima/mtp complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 mtp status", ["ima/mtp complete"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# filtered_dfs.append(v3) + +# Design stage 1 +# design1 = get_df(df, "", ["done"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") +# filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +# full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full Lodgement") +# filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc", ["done"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +# post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") +# filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +# att_ns = df[ +# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file From 843d30f060bb6456b3312df4eec8e291a86def88 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 24 Jul 2025 15:50:48 +0000 Subject: [PATCH 82/93] save current progress --- etl/month_end_automation_wave_2_no_14.py | 169 +++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 etl/month_end_automation_wave_2_no_14.py diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py new file mode 100644 index 0000000..9f4e83b --- /dev/null +++ b/etl/month_end_automation_wave_2_no_14.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Decent Homes Stonewater - Operations +board_ids = ["9319118237"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +# att = get_df(df, "att", ["completed"], "ATT") +# filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +filtered_dfs.append(v3) + +v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "phase 1 invoice status (lodgement)".lower(), ["to be invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc", ["completed & uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["completed & uploaded"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +# att_ns = df[ +# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file From 1fef7bf29cd28746daa50c54dfd320afea543173 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 24 Jul 2025 16:00:29 +0000 Subject: [PATCH 83/93] save --- etl/month_end_automation_wave_2_no_15.py | 169 +++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 etl/month_end_automation_wave_2_no_15.py diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py new file mode 100644 index 0000000..c4e3613 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_15.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# WCHG SHDF 2.1 Mansard +board_ids = ["5636990610"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra status", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# PRE- ATT +att = get_df(df, "pre att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status (ioe mtp)".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# filtered_dfs.append(v3) + +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "tm phase 1 invoice satus (lodgment)".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post-att status", ["uploaded", "completed"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed", "uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file From 80758b35f8c0a39bafd83e0aadc075e1c04b717e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 25 Jul 2025 11:26:14 +0000 Subject: [PATCH 84/93] next one --- etl/month_end_automation_wave_2_no_16.py | 169 ++++++++++++++++++++++ etl/month_end_automation_wave_3_layout.py | 154 +++++++++----------- 2 files changed, 235 insertions(+), 88 deletions(-) create mode 100644 etl/month_end_automation_wave_2_no_16.py diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py new file mode 100644 index 0000000..0c6d3c1 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_16.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA SHDF Wave 3 On Hold +board_ids = ["6946967610"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra", ["completed"], "RA") +filtered_dfs.append(ra) + + +# PRE- ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# filtered_dfs.append(v3) + +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) + +# Design stage 1 +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "lodg. phase 1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "lodged epc", ["complete", "complete & lodged",], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["done", "post att complete"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 586b6da..e393890 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -9,11 +9,15 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) -# WCHG Walkups-Operations -board_ids = ["9349630181"] +board_ids = [ + # "9349630181", # WCHG Walkups-Operations + "8829428746", # 2502 Accent Housing + +] for board in tqdm(board_ids): + print(f"working on board {board}") board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] col_id_map = {col["title"].lower(): col["id"] for col in columns} @@ -68,81 +72,69 @@ df = pd.DataFrame(all_records) filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + else: + print(f"failed to find {column_name}") + + # RA -ra = df[ - df["ra invoicing status"].str.lower().isin(["to invoice"]) -].copy() -ra["job_type"] = "RA" -filtered_dfs.append(ra) +ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") +if ra is not None: + filtered_dfs.append(ra) -# ATT -att = df[ - df["pre att invoicing status"].str.lower().isin(["to invoice"]) -].copy() -att["job_type"] = "ATT" -filtered_dfs.append(att) +att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") +if att is not None: + filtered_dfs.append(att) -# V1 Coordination -v1 = df[ - df["mtp invoicing status"].str.lower().isin(["(v1) ioe/mtp to invoice"]) -].copy() +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "V3 Coordination") +if modeling is not None: + filtered_dfs.append(modeling) -v1["job_type"] = "V1 Coordination" -filtered_dfs.append(v1) +v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "V1 Coordination") +if v1 is not None: + filtered_dfs.append(v1) -# V2 Coordination -v2 = df[ - df["mtp invoicing status"].str.lower().isin(["(v2) ioe/mtp to invoice"]) -].copy() +v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "V2 Coordination") +if v2 is not None: + filtered_dfs.append(v2) -v2["job_type"] = "V2 Coordination" -filtered_dfs.append(v2) +v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "V3 Coordination") +if v3 is not None: + filtered_dfs.append(v3) -# V3 Coordination -v3 = df[ - df["mtp invoicing status"].str.lower().isin(["(v3) ioe/mtp to invoice"]) -].copy() - -v3["job_type"] = "V3 Coordination" -filtered_dfs.append(v3) - -# Design stage 1 -design1 = df[ - df["design invoicing status"].str.lower().isin(["to invoice"]) -].copy() -design1["job_type"] = "Design" -filtered_dfs.append(design1) +design1 = get_df(df, "design invoicing status", ["to invoice"], "Design") +if design1 is not None: + filtered_dfs.append(design1) # Design Revision revision_letter = ['a', 'b', 'c', 'd'] for letter in revision_letter: - design2 = df[ - df["design revision invoice"].str.lower().isin([f"rev. {letter} to invoice"]) - ].copy() - design2["job_type"] = "Design Revision" - filtered_dfs.append(design2) + design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") + if design is not None: + filtered_dfs.append(design) # Lodgement Phase 1 -lodg1 = df[ - df["lodgement phase 1 invoicing status"].str.lower().isin(["to invoice"]) -].copy() -lodg1["job_type"] = "Lodgement Phase 1" -filtered_dfs.append(lodg1) +lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") +if lodg1 is not None: + filtered_dfs(lodg1) # Full Lodgement Phase -lodg2 = df[ - df["full lodgement invoicing status"].str.lower().isin(["to invoice"]) -].copy() -lodg2["job_type"] = "Full Lodgement" -filtered_dfs.append(lodg2) +lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full Lodgement") +if lodg2 is not None: + filtered_dfs.append(lodg2) # POST EPC -post_epc = df[ - df["post epc & eval. invoicing status"].str.lower().isin(["to invoice"]) -].copy() -post_epc["job_type"] = "POST EPC" -filtered_dfs.append(post_epc) +post_epc = get_df(df, "post epc & eval. invoicing status", ["to invoice"], "POST EPC") +if post_epc is not None: + filtered_dfs.append(post_epc) # # POST EPR, not right now @@ -152,45 +144,31 @@ filtered_dfs.append(post_epc) # post_epr["job_type"] = "POST ATT" # filtered_dfs.append(post_epr) - -# Post ATT -post_att = df[ - df["post att invoicing status"].str.lower().isin(["to invoice"]) -].copy() -post_att["job_type"] = "POST ATT" -filtered_dfs.append(post_att) - +post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") +if post_att is not None: + filtered_dfs.append(post_epc) # Retrofit Evaluation -retro = df[ - df["rc stage 2 invoicing status"].str.lower().isin(["to invoice"]) -].copy() -retro["job_type"] = "RC stage 2" -filtered_dfs.append(retro) +rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "RC stage 2") +if rc is not None: + filtered_dfs.append(rc) # RA NO Show -ra_ns = df[ - df["ra no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) -].copy() -ra_ns["job_type"] = "RA NO SHOW" -filtered_dfs.append(ra_ns) +ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") +if ra_ns is not None: + filtered_dfs.append(ra_ns) # ATT NO Show -att_ns = df[ - df["pre att no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) -].copy() -att_ns["job_type"] = "ATT NO SHOW" -filtered_dfs.append(att_ns) +att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") +if att_ns is not None: + filtered_dfs.append(att_ns) # Post visit no show -#post work no show invoice -epc_ns = df[ - df["post works no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) -].copy() -epc_ns["job_type"] = "EPC NO SHOW" -filtered_dfs.append(epc_ns) +epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "EPC NO SHOW") +if epc_ns is not None: + filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) From 7ee4bf53109f0dff476c18ba0e56b8a6bbeea9f2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 25 Jul 2025 11:36:32 +0000 Subject: [PATCH 85/93] wave 3 layout --- etl/month_end_automation_wave_3_layout.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index e393890..9a051e9 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -11,8 +11,9 @@ monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozN monday = MondayClient(monday_key) board_ids = [ # "9349630181", # WCHG Walkups-Operations - "8829428746", # 2502 Accent Housing - + # "8829428746", # 2502 Accent Housing + # "8830772914", # "L&Q London" + "9601691730", # Cardo Wales & West - Wave 3 ] From d1483fe07231d448e04223d90a41c282e25aeb05 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 28 Jul 2025 16:14:18 +0000 Subject: [PATCH 86/93] no show re write --- etl/month_end_automation_wave_2_layout.py | 80 ++++++++++++++++++----- etl/month_end_automation_wave_2_no_10.py | 6 +- etl/month_end_automation_wave_2_no_11.py | 3 +- etl/month_end_automation_wave_2_no_12.py | 6 +- etl/month_end_automation_wave_2_no_13.py | 3 +- etl/month_end_automation_wave_2_no_14.py | 6 +- etl/month_end_automation_wave_2_no_15.py | 9 ++- etl/month_end_automation_wave_2_no_16.py | 6 +- etl/month_end_automation_wave_2_no_3.py | 9 ++- etl/month_end_automation_wave_2_no_4.py | 9 ++- etl/month_end_automation_wave_2_no_5.py | 9 ++- etl/month_end_automation_wave_2_no_7.py | 9 ++- etl/month_end_automation_wave_2_no_8.py | 73 ++++++++++++++++++--- etl/month_end_automation_wave_2_no_9.py | 6 +- 14 files changed, 179 insertions(+), 55 deletions(-) diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 843e2fa..174fe0b 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -9,9 +9,26 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) -# NCHK SHDF Westville Wave 1 & 2 +# NCHA SHDF Westville Wave 1 & 2 board_ids = ["3900434153"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 207.65, 101, 186.4, 98, 98, + 450, 150, 163, 135, 120, + 45, 90.5, 40, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,55 +104,69 @@ filtered_dfs.append(att) v1 = df[ df["v1 coordination status"].str.lower().isin(["rc complete"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] v2 = _[_["v2 dc/ima/pas"] > 0].copy() -v2["job_type"] = "V2 Coordination" +v2["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(v2) # V3 Coordination v3 = df[ df["v3 invoiced"].str.lower().isin(["to be invoiced"]) ].copy() -v3["job_type"] = "V3 Coordination" +v3["job_type"] = "Coordination Stage 1 v3 remodel" filtered_dfs.append(v3) -# Design stage 1 +# Coordination stage 2 +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design type archietype design1 = df[ - df["design invoice status"].str.lower().isin(["to invoice"]) + (df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"])) ].copy() -design1["job_type"] = "Design" +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) -# Design stage 2 +# design type reptitive +design1 = df[ + (df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"]) +].copy() +design1["job_type"] = "Design Repetitive" +filtered_dfs.append(design1) + +# Design stage revisions design2 = df[ df["revision 2 design invoice"].str.lower().isin(["to invoice"]) ].copy() -design2["job_type"] = "Design Revision" +design2["job_type"] = "Design Repetitive" filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = df[ df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) ].copy() -lodg1["job_type"] = "Lodgement Phase 1" +lodg1["job_type"] = "Lodgement phase 1" filtered_dfs.append(lodg1) # Full Lodgement Phase lodg2 = df[ df["full lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() -lodg2["job_type"] = "Full Lodgement" +lodg2["job_type"] = "Full Lodgement phase 2" filtered_dfs.append(lodg2) # POST EPC post_epc = df[ df["post-epc status"].str.lower().isin(["epc pics uploaded"]) ].copy() -post_epc["job_type"] = "POST EPC" +post_epc["job_type"] = "Post EPC" filtered_dfs.append(post_epc) @@ -143,7 +174,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "Post ATT" filtered_dfs.append(post_epr) @@ -165,7 +196,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -174,7 +206,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -182,11 +215,22 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_csv(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.csv', index=False) diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index 87580f2..d779e45 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -144,7 +144,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] !=0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -152,7 +153,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index babf1f8..2eeb76e 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -152,7 +152,8 @@ filtered_dfs.append(retro) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 261ed2f..fbe4bf0 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -141,7 +141,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -157,7 +158,8 @@ filtered_dfs.append(ra_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index 412c6f7..30a09bf 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -140,7 +140,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 9f4e83b..733be63 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -159,7 +160,8 @@ filtered_dfs.append(ra_ns) # Post visit no show epc_ns = df[ - df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index c4e3613..9348fe4 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -159,7 +161,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 0c6d3c1..ce43999 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 8a7ca4c..56b2efd 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -164,7 +164,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df[" ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -173,7 +174,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -181,7 +183,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 6fe0d90..4678aba 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -160,7 +160,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -168,7 +169,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -176,7 +178,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index daaa8b0..f244dc5 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -160,7 +160,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -168,7 +169,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999) + (df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)) & + (df["pre att no show evidence"] ! = 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -176,7 +178,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0 ) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index fb15205..d834847 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -159,7 +161,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0 ) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index ddd2368..8951e71 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # NCHA SHDF 2.1 SBS board_ids = ["8668578700"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -67,11 +84,12 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name + if job_name: + _["job_type"] = job_name return _ @@ -87,21 +105,43 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "v1 coordination status".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination # v3 = get_df(df, "") # # filtered_dfs.append(v3) -# Design stage 1 -design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") +# TODO check with Kevin [Coordination stage 2, design complex and revision] +# Corodination stage 2 +cors2 = get_df(df, "rc stg. 2", ["to invoice"], "Coordination Stage 2") +filtered_dfs.append(cors2) + +# Design Archtype Complex +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["archetype (complex)"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Complex") filtered_dfs.append(design1) +# Design Archtype Simple +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["archetype (simple)"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Simple") +filtered_dfs.append(design1) + +# Design Repitive Simple +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["Design Repetitive Simple"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repetitive Simple") +filtered_dfs.append(design1) + +# TOOD Kevin: +# Check with Design checks and check with cor dination stage 2 checks + # Design revision # design2 = get_df(df, "design revision invoice", [ # "Rev. A to invoice".lower(), @@ -120,7 +160,7 @@ lodg1 = get_df(df, "phase 1 to be invoiced".lower(), ["phase 1 to be invoiced"], filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement") +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -146,7 +186,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -154,7 +195,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] !=0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -169,4 +211,15 @@ filtered_dfs.append(att_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_csv(f'NCHA SHDF 2.1 SBS_{timestamp}.csv', index=False) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index ca198f5..f295ca6 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999))) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) From 42dca1c88a94655f9d1c06419c3cfd84cffb249a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 28 Jul 2025 16:21:05 +0000 Subject: [PATCH 87/93] added corodination --- etl/month_end_automation_wave_2_layout.py | 2 +- etl/month_end_automation_wave_2_no_10.py | 7 +++++++ etl/month_end_automation_wave_2_no_11.py | 7 +++++++ etl/month_end_automation_wave_2_no_12.py | 7 +++++++ etl/month_end_automation_wave_2_no_13.py | 7 +++++++ etl/month_end_automation_wave_2_no_14.py | 7 +++++++ etl/month_end_automation_wave_2_no_15.py | 8 ++++++++ etl/month_end_automation_wave_2_no_16.py | 7 +++++++ etl/month_end_automation_wave_2_no_3.py | 7 +++++++ etl/month_end_automation_wave_2_no_4.py | 7 +++++++ etl/month_end_automation_wave_2_no_5.py | 7 +++++++ etl/month_end_automation_wave_2_no_6.py | 7 +++++++ etl/month_end_automation_wave_2_no_7.py | 7 +++++++ etl/month_end_automation_wave_2_no_8.py | 1 - etl/month_end_automation_wave_2_no_9.py | 7 +++++++ etl/month_end_automation_wave_3_layout.py | 7 +++++++ 16 files changed, 100 insertions(+), 2 deletions(-) diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 174fe0b..f93f8f3 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -120,7 +120,7 @@ v3 = df[ v3["job_type"] = "Coordination Stage 1 v3 remodel" filtered_dfs.append(v3) -# Coordination stage 2 +# Coordination stage 2 Please complete cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) ] diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index d779e45..29e5845 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -97,6 +97,13 @@ filtered_dfs.append(v1) # v3 = get_df(df, "") # # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index 2eeb76e..da17811 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -97,6 +97,13 @@ filtered_dfs.append(v2) # v3 = get_df(df, "") # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") # filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index fbe4bf0..0de0630 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -97,6 +97,13 @@ filtered_dfs.append(v2) # v3 = get_df(df, "") # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") # filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index 30a09bf..f0bfe90 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -96,6 +96,13 @@ filtered_dfs.append(v2) # v3 = get_df(df, "") # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") # filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 733be63..e3286ea 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -99,6 +99,13 @@ filtered_dfs.append(v3) v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = get_df(df, "design invoice status", ["to invoice"], "Design") filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index 9348fe4..bcdc46b 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -96,9 +96,17 @@ filtered_dfs.append(v1) # v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") # filtered_dfs.append(v3) + # v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = get_df(df, "design invoice status", ["to invoice"], "Design") filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index ce43999..8b83fe9 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -99,6 +99,13 @@ filtered_dfs.append(v1) # v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 # design1 = get_df(df, "design invoice status", ["to invoice"], "Design") # filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 56b2efd..90253d5 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -102,6 +102,13 @@ filtered_dfs.append(_) # v3["job_type"] = "V3 Coordination" # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = df[ df["design invoice"].str.lower().isin(["complete pending rc"]) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 4678aba..b750850 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -100,6 +100,13 @@ v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() v3["job_type"] = "V3 Coordination" filtered_dfs.append(_) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = df[ df["design invoice status"].str.lower().isin(["complete", "to invoice"]) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index f244dc5..5ed035b 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -100,6 +100,13 @@ filtered_dfs.append(v1) # v3["job_type"] = "V3 Coordination" # filtered_dfs.append(_) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = df[ df["retrofit design status"].str.lower().isin(["to invoice"]) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index d069018..1b149ae 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -102,6 +102,13 @@ filtered_dfs.append(v1) # v3["job_type"] = "V3 Coordination" # filtered_dfs.append(_) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = get_df(df, "retrofit design status", ["completed"], "Design") filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index d834847..46d0a33 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -99,6 +99,13 @@ filtered_dfs.append(v2) # v3 = get_df(df, "") # # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 design1 = get_df(df, "design invoice status", ["to invoice"], "Design") filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 8951e71..653c543 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -116,7 +116,6 @@ filtered_dfs.append(v2) # v3 = get_df(df, "") # # filtered_dfs.append(v3) -# TODO check with Kevin [Coordination stage 2, design complex and revision] # Corodination stage 2 cors2 = get_df(df, "rc stg. 2", ["to invoice"], "Coordination Stage 2") filtered_dfs.append(cors2) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index f295ca6..a24b6e7 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -98,6 +98,13 @@ filtered_dfs.append(v1) # v3 = get_df(df, "") # # filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + # Design stage 1 # design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") # filtered_dfs.append(design1) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 9a051e9..3b5f53f 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -111,6 +111,13 @@ v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "V3 Coordin if v3 is not None: filtered_dfs.append(v3) +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + design1 = get_df(df, "design invoicing status", ["to invoice"], "Design") if design1 is not None: filtered_dfs.append(design1) From c297a87776a9ca1c87a47a8247200f5a72b9585c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 11:41:55 +0000 Subject: [PATCH 88/93] next board --- etl/month_end_automation_wave_2_layout.py | 12 ++-- etl/month_end_automation_wave_2_no_16.py | 68 ++++++++++++++++------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index f93f8f3..5d208e2 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -16,13 +16,13 @@ rate_card_data = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPC", "Post ATT", "retrofit evaluation", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show" ], "rate": [ 207.65, 101, 186.4, 98, 98, 450, 150, 163, 135, 120, - 45, 90.5, 40, + "Mariaane Please tell me", 45, 90.5, 40, 25, 25, 25 ] } @@ -122,8 +122,8 @@ filtered_dfs.append(v3) # Coordination stage 2 Please complete cors2 = df[ - df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -] + df["rc stg. 2"].str.lower().isin(["to invoice"]) +].copy() cors2["joby_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) @@ -174,7 +174,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "Post ATT" +post_epr["job_type"] = "Post EPR" filtered_dfs.append(post_epr) @@ -233,4 +233,4 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_csv(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.csv', index=False) +combined_with_rates[attribute].to_excel(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx', index=False) \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 8b83fe9..be52ffd 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # NCHA SHDF Wave 3 On Hold board_ids = ["6946967610"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 207.65, 101, 186.4, 98, 98, + 450, 150, 163, 135, 120, + "Marianne EPR Please", 45, 90.5, 40, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -85,32 +102,35 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "Coordination Stage 1 v2 remodel") # filtered_dfs.append(v2) # # V3 Coordination -# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# v3 = get_df(df, "v3 rc status", ["uploaded"], "Coordination Stage 1 v3 remode") # filtered_dfs.append(v3) # v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 +# Design Archetype # design1 = get_df(df, "design invoice status", ["to invoice"], "Design") # filtered_dfs.append(design1) -# Design revision + +# Design Repetitive + +# Design Revision # design2 = get_df(df, "design revision invoice", [ # "Rev. A to invoice".lower(), # "Rev. B to invoice".lower(), @@ -125,20 +145,20 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC -post_epc = get_df(df, "lodged epc", ["complete", "complete & lodged",], "POST EPC") +post_epc = get_df(df, "lodged epc", ["complete", "complete & lodged",], "Post EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["lodged epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att", ["done", "post att complete"], "POST ATT") @@ -175,4 +195,14 @@ filtered_dfs.append(att_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False) \ No newline at end of file From 9c7e7ecdd94056d3c5b8be055a6b8465cc402c27 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 12:01:53 +0000 Subject: [PATCH 89/93] save --- etl/month_end_automation_wave_2_no_8.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 653c543..62950e3 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -15,13 +15,13 @@ rate_card_data = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPC", "Post ATT", "retrofit evaluation", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show" ], "rate": [ 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, - 120, 85, 125, 60, + 120, "Post EPR Please Marianne", 85, 125, 60, 25, 25, 25 ] } @@ -138,9 +138,6 @@ design1 = get_df(design1, "design type", ["Design Repetitive Simple"]) design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repetitive Simple") filtered_dfs.append(design1) -# TOOD Kevin: -# Check with Design checks and check with cor dination stage 2 checks - # Design revision # design2 = get_df(df, "design revision invoice", [ # "Rev. A to invoice".lower(), @@ -167,12 +164,12 @@ post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT") @@ -221,4 +218,4 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_csv(f'NCHA SHDF 2.1 SBS_{timestamp}.csv', index=False) +combined_with_rates[attribute].to_excel(f'NCHA SHDF 2.1 SBS_{timestamp}.xlsx', index=False) From a751b8d3613aadb7c0be101414935a4ac23cd4c5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 13:13:45 +0000 Subject: [PATCH 90/93] month end --- etl/month_end_automation_wave_2_no_10.py | 63 +++++++++++++----- etl/month_end_automation_wave_2_no_5.py | 83 +++++++++++++++++------- etl/month_end_automation_wave_2_no_6.py | 61 ++++++++++++----- etl/month_end_automation_wave_2_no_9.py | 51 +++++++++++---- 4 files changed, 189 insertions(+), 69 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index 29e5845..acba26b 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -11,6 +11,24 @@ monday = MondayClient(monday_key) # NCHA Derbyshire Dales (DDDCC) SHDF board_ids = ["6947307148"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -86,7 +104,7 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination @@ -98,15 +116,15 @@ filtered_dfs.append(v1) # # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 -design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") -filtered_dfs.append(design1) +# # Design stage 1 +# design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") +# filtered_dfs.append(design1) # Design revision # design2 = get_df(df, "design revision invoice", [ @@ -123,7 +141,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -131,12 +149,12 @@ post_epc = get_df(df, "lodged epc", ["complete"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["lodged epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "Post EPR" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att", ["done"], "POST ATT") @@ -176,4 +194,15 @@ filtered_dfs.append(att_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA Derbyshire Dales (DDDCC) SHDF_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index 5ed035b..5d40b55 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -12,6 +12,23 @@ monday = MondayClient(monday_key) # ECO 4 NCHA Almshouses Operations board_ids = ["9136254638"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,39 +104,50 @@ filtered_dfs.append(att) v1 = df[ df["coordination status"].str.lower().isin(["ioe/mtp complete"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination # _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() -# _["job_type"] = "V2 Coordination" +# _["job_type"] = "Coordination Stage 1 v2 remodel" # filtered_dfs.append(_) # V3 Coordination # v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() -# v3["job_type"] = "V3 Coordination" +# v3["job_type"] = "Coordination Stage 1 v3 remodel" # filtered_dfs.append(_) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice",]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 design1 = df[ df["retrofit design status"].str.lower().isin(["to invoice"]) ].copy() -design1["job_type"] = "Design" +design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) # Design stage 2 -# design2 = df[ -# df[""].str.lower().isin(["to invoice"]) -# ].copy() -# design2["job_type"] = "Design Revision" -# filtered_dfs.append(design2) +design2 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design2 = design2[design2["design type"].str.lower().isin(["repetitive"])].copy() +design2["job_type"] = "Design Repetitive" +filtered_dfs.append(design2) + +# Design revision +design2 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design2 = design2[design2["design revision"].str.lower().isin(["A", "B", "C"])].copy() +design2["job_type"] = "Design Repetitive" +filtered_dfs.append(design2) + # Lodgement Phase 1 # lodg1 = df[ @@ -132,7 +160,7 @@ filtered_dfs.append(design1) _ = df[ df["trustmark lodgement"].str.lower().isin(["done"]) ].copy() -_["job_type"] = "Full Lodgement" +_["job_type"] = "Full lodgement phase 2" filtered_dfs.append(_) # POST EPC @@ -143,12 +171,12 @@ post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = df[ @@ -177,7 +205,7 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ (df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)) & - (df["pre att no show evidence"] ! = 0) + (df["pre att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -193,4 +221,13 @@ filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'ECO 4 NCHA Almshouses Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 1b149ae..69e9e53 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -12,6 +12,22 @@ monday = MondayClient(monday_key) #ECO 4 Wates Operations board_ids = ["9520779048"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -89,25 +105,25 @@ v1 = get_df(df, "Coordination Status IOE/MTP".lower(), [ "(V1) IOE/MTP Complete".lower(), "(V2) IOE/MTP Complete".lower(), "(V3) IOE/MTP Complete".lower(), -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination # _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() -# _["job_type"] = "V2 Coordination" +# _["job_type"] = "Coordination Stage 1 v2 remodel" # filtered_dfs.append(_) # V3 Coordination # v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() -# v3["job_type"] = "V3 Coordination" +# v3["job_type"] = "Coordination Stage 1 v3 remodel" # filtered_dfs.append(_) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2 status"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 design1 = get_df(df, "retrofit design status", ["completed"], "Design") @@ -119,7 +135,7 @@ design2 = get_df(df, "design revision invoice", [ "Rev. B to invoice".lower(), "Rev. C to invoice".lower(), "Rev. D to invoice".lower(), -], "Design Revision") +], "Design Repetitive") filtered_dfs.append(design2) # Lodgement Phase 1 @@ -130,7 +146,7 @@ filtered_dfs.append(design2) # filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "full lodgement", ["completed"], "Full Lodgement") +full_lodgement = get_df(df, "full lodgement", ["completed"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -138,12 +154,12 @@ post_epc = get_df(df, "post epc & evaluation status", ["uploaded"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc & evaluation status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att status", ["uploaded"], "POST ATT") @@ -151,7 +167,7 @@ filtered_dfs.append(post_att) # Retrofit Evaluation -retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +retro = get_df(df, "post epc & evaluation status", ["uploaded"], "Retrofit Evaluation") filtered_dfs.append(retro) # RA NO Show @@ -179,4 +195,13 @@ filtered_dfs.append(retro) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'ECO 4 Wates Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index a24b6e7..4946193 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # NCHA Almshouses board_ids = ["5423364294"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,23 +104,24 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status (mtp)".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel") # filtered_dfs.append(v2) # # V3 Coordination +# Coordination Stage 1 v3 remode # v3 = get_df(df, "") # # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +].copy() +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") @@ -124,7 +142,7 @@ filtered_dfs.append(v1) # Full Lodgement Phase -full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement") +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -159,7 +177,7 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - (df(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999))) & + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" @@ -170,9 +188,20 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC no show" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA Almshouses_{timestamp}.xlsx', index=False) From 444f2836160a4da569dbe8e7fed0559bd2152892 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 14:00:00 +0000 Subject: [PATCH 91/93] save --- etl/month_end_automation_wave_2_no_11.py | 61 +++++++++++++------ etl/month_end_automation_wave_2_no_7.py | 75 +++++++++++++++++------- 2 files changed, 96 insertions(+), 40 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index da17811..4ea345f 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # Northumberland LAD2 & HUG2 board_ids = ["5121300882"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -66,7 +83,7 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() @@ -86,11 +103,11 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "lite ima status".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "ima-mtp status", ["ima-mtp completed"], "V2 Coordination") +v2 = get_df(df, "ima-mtp status", ["ima-mtp completed"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -98,11 +115,11 @@ filtered_dfs.append(v2) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") @@ -123,7 +140,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement") +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -131,20 +148,17 @@ post_epc = get_df(df, "post-epc status", ["uploaded & completed", "to invoice"], filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att status", ["uploaded & completed", "to invoice"], "POST ATT") filtered_dfs.append(post_att) -post_att = get_df(df, "post-test status", ["complete"], "POST ATT") -filtered_dfs.append(post_att) - # Retrofit Evaluation retro = get_df(df, "retrofit evaluation", ["done", "to invoice"], "Retrofit Evaluation") filtered_dfs.append(retro) @@ -175,4 +189,13 @@ filtered_dfs.append(att_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Northumberland LAD2 & HUG2_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 46d0a33..cbb3e2b 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -12,6 +12,23 @@ monday = MondayClient(monday_key) #Home Group Wave 2SP+ board_ids = ["4254419092"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -68,11 +85,12 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name + if job_name: + _["job_type"] = job_name return _ @@ -86,13 +104,13 @@ att = get_df(df, "pre-att", ["completed"], "ATT") filtered_dfs.append(att) # V1 Coordination -v1 = get_df(df, "osmosis rc status".lower(), [ +v1 = get_df(df, "osmosis rc status".lower(), [ "rc completed", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "v2 ioe mtp", ["completed"], "V2 Coordination") +v2 = get_df(df, "v2 ioe mtp", ["completed"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -100,14 +118,20 @@ filtered_dfs.append(v2) # # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 -design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# Design stage Archetype +design1 = get_df(df, "design invoice status", ["to invoice"]) +design1 = get_df(design1, "design type for invoicing", ["archetype"], "Design Archetype") +filtered_dfs.append(design1) + +# Design stage Repetitive +design1 = get_df(df, "design invoice status", ["to invoice"]) +design1 = get_df(design1, "design type for invoicing", ["repetitive"], "Design Repetitive") filtered_dfs.append(design1) # Design revision @@ -120,11 +144,11 @@ filtered_dfs.append(design1) # filtered_dfs.append(design2) # Lodgement Phase 1 -lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement Phase 1") +lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement phase 1") filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -132,12 +156,12 @@ post_epc = get_df(df, "post-epc status", ["complete & uploaded"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT") @@ -176,4 +200,13 @@ filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) From 4255b3036b0c1fb5f2258cce9a55c669c0bddfa9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 15:51:25 +0000 Subject: [PATCH 92/93] post epr --- etl/month_end_automation_wave_2_no_10.py | 2 +- etl/month_end_automation_wave_2_no_11.py | 4 +- etl/month_end_automation_wave_2_no_12.py | 58 +++++--- etl/month_end_automation_wave_2_no_13.py | 58 +++++--- etl/month_end_automation_wave_2_no_14.py | 4 +- etl/month_end_automation_wave_2_no_15.py | 59 +++++--- etl/month_end_automation_wave_2_no_16.py | 2 +- etl/month_end_automation_wave_2_no_3.py | 84 ++++++++---- etl/month_end_automation_wave_2_no_4.py | 75 +++++++--- etl/month_end_automation_wave_2_no_5.py | 4 +- etl/month_end_automation_wave_2_no_6.py | 4 +- etl/month_end_automation_wave_2_no_7.py | 4 +- etl/month_end_automation_wave_2_no_8.py | 4 +- etl/month_end_automation_wave_2_no_9.py | 2 +- etl/month_end_automation_wave_3_layout.py | 158 +++++++++++++--------- 15 files changed, 347 insertions(+), 175 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index acba26b..d854fdc 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -189,7 +189,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index 4ea345f..f05ef13 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -152,7 +152,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -184,7 +184,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 0de0630..962eb46 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # Shropshire Council HUG2 board_ids = ["4718185486"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -86,11 +103,11 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "ima lite invoiced".lower(), [ "to invoice", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "coordination v2 invoiced", ["to invoice"], "V2 Coordination") +v2 = get_df(df, "coordination v2 invoiced", ["to invoice"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -98,11 +115,11 @@ filtered_dfs.append(v2) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2 invoice"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") @@ -123,7 +140,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -131,12 +148,12 @@ post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") @@ -168,9 +185,18 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Shropshire Council HUG2_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index f0bfe90..d4a6a1d 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # Stonewater SHDF 3.0 - Operations board_ids = ["6222522864"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -85,11 +102,11 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status (mtp)".lower(), [ "ima/mtp complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "v2 mtp status", ["ima/mtp complete"], "V2 Coordination") +v2 = get_df(df, "v2 mtp status", ["ima/mtp complete"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -97,11 +114,11 @@ filtered_dfs.append(v2) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") @@ -122,7 +139,7 @@ filtered_dfs.append(v2) # Full Lodgement Phase -# full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full Lodgement") +# full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2") # filtered_dfs.append(full_lodgement) # POST EPC @@ -130,12 +147,12 @@ post_epc = get_df(df, "post epc", ["done"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) # Post ATT # post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") @@ -166,9 +183,18 @@ filtered_dfs.append(ra_ns) # epc_ns = df[ # df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Stonewater SHDF 3.0 - Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index e3286ea..9203f2b 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -137,7 +137,7 @@ filtered_dfs.append(post_epc) # post_epr = df[ # df["post-epc status"].str.lower().isin(["post epr completed"]) # ].copy() -# post_epr["job_type"] = "POST ATT" +# post_epr["job_type"] = "POST epr" # filtered_dfs.append(post_epr) # Post ATT @@ -170,7 +170,7 @@ epc_ns = df[ (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & (df["post epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index bcdc46b..3686198 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -11,6 +11,24 @@ monday = MondayClient(monday_key) # WCHG SHDF 2.1 Mansard board_ids = ["5636990610"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -85,7 +103,7 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status (ioe mtp)".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination @@ -101,15 +119,15 @@ filtered_dfs.append(v1) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 -design1 = get_df(df, "design invoice status", ["to invoice"], "Design") -filtered_dfs.append(design1) +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) # Design revision # design2 = get_df(df, "design revision invoice", [ @@ -126,7 +144,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -135,11 +153,11 @@ filtered_dfs.append(post_epc) # # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post-att status", ["uploaded", "completed"], "POST ATT") @@ -172,9 +190,18 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index be52ffd..0af8bfe 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -190,7 +190,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 90253d5..6e497c5 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -12,6 +12,23 @@ monday = MondayClient(monday_key) # Platform Housing W2 (in use) board_ids = ["4796290860"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,41 +104,45 @@ filtered_dfs.append(att) v1 = df[ df["coordination status"].str.lower().isin(["ima/mtp completed"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 mtp status"].fillna('').str.lower().isin(['v2 ima-mtp completed', 'v2 completed'])].copy() -_["job_type"] = "V2 Coordination" +_["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(_) # V3 Coordination # v3 = df[ # df["v3 invoiced"].str.lower().isin(["to be invoiced"]) # ].copy() -# v3["job_type"] = "V3 Coordination" +# v3["job_type"] = "Coordination Stage 1 v3 remodel" # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 +# Design Archetype design1 = df[ df["design invoice"].str.lower().isin(["complete pending rc"]) ].copy() -design1["job_type"] = "Design" +design1 = design1[design1["design type for invoicing"].str.lower().isin(['archetype'])].copy() +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) -# Design stage 2 -# design2 = df[ -# df["revision 2 design invoice"].str.lower().isin(["to invoice"]) -# ].copy() -# design2["job_type"] = "Design Revision" -# filtered_dfs.append(design2) +# Design Repetitive +design1 = df[ + df["design invoice"].str.lower().isin(["complete pending rc"]) +].copy() +design1 = design1[design1["design type for invoicing"].str.lower().isin(['repetitive'])].copy() +design1["job_type"] = "Design repetitive" +filtered_dfs.append(design1) + +# Design Revision # Lodgement Phase 1 lodg1 = df[ @@ -134,7 +155,7 @@ filtered_dfs.append(lodg1) lodg2 = df[ df["lodgement invoice status (lodgement)"].str.lower().isin(["to invoice"]) ].copy() -lodg2["job_type"] = "Full Lodgement" +lodg2["job_type"] = "Full lodgement phase 2" filtered_dfs.append(lodg2) # POST EPC @@ -145,13 +166,12 @@ post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) - +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) # Post ATT @@ -172,13 +192,12 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & - (df[" ra no show evidence"] != 0) + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) - # ATT NO Show att_ns = df[ (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & @@ -193,9 +212,18 @@ epc_ns = df[ (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & (df["post epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "Post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Platform Housing W2 (in use)_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index b750850..1c17c7e 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -12,6 +12,22 @@ monday = MondayClient(monday_key) # Stonewater (in use) board_ids = ["3584401309"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,34 +103,43 @@ filtered_dfs.append(att) v1 = df[ df["v1 coordination status (ioe,mtp)"].str.lower().isin(["rc complete"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() -_["job_type"] = "V2 Coordination" +_["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(_) # V3 Coordination v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() -v3["job_type"] = "V3 Coordination" +v3["job_type"] = "Coordination Stage 1 v3 remodel" filtered_dfs.append(_) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 +# Design Archetype design1 = df[ df["design invoice status"].str.lower().isin(["complete", "to invoice"]) ].copy() -design1["job_type"] = "Design" +design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) -# Design stage 2 +# Design Repetitive +design1 = df[ + df["design invoice status"].str.lower().isin(["complete", "to invoice"]) +].copy() +design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy() +design1["job_type"] = "Design Repetitive" +filtered_dfs.append(design1) + +# Design revision # design2 = df[ # df[""].str.lower().isin(["to invoice"]) # ].copy() @@ -132,7 +157,7 @@ filtered_dfs.append(lodg1) _ = df[ df["lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() -_["job_type"] = "Full Lodgement" +_["job_type"] = "Full lodgement phase 2" filtered_dfs.append(_) # POST EPC @@ -143,12 +168,12 @@ post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) # Post ATT post_att = df[ @@ -188,9 +213,19 @@ epc_ns = df[ (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & (df["post epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Stonewater - (in use)_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index 5d40b55..f26d3b0 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -175,7 +175,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -216,7 +216,7 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0 ) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 69e9e53..0ad996f 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -158,7 +158,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post epc & evaluation status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -190,7 +190,7 @@ filtered_dfs.append(retro) # epc_ns = df[ # df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index cbb3e2b..0cc1749 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -160,7 +160,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -195,7 +195,7 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0 ) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 62950e3..d8f0a35 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -168,7 +168,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -202,7 +202,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index 4946193..15bf2fb 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -154,7 +154,7 @@ filtered_dfs.append(full_lodgement) # post_epr = df[ # df["post-epc status"].str.lower().isin(["post epr completed"]) # ].copy() -# post_epr["job_type"] = "POST ATT" +# post_epr["job_type"] = "POST epr" # filtered_dfs.append(post_epr) # Post ATT diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 3b5f53f..7e273f2 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -12,10 +12,27 @@ monday = MondayClient(monday_key) board_ids = [ # "9349630181", # WCHG Walkups-Operations # "8829428746", # 2502 Accent Housing - # "8830772914", # "L&Q London" - "9601691730", # Cardo Wales & West - Wave 3 + "8830772914", # "L&Q London" + # "9601691730", # Cardo Wales & West - Wave 3 ] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25, "Mariann please input full cost mtp", "Marianne please input measure modelling" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): print(f"working on board {board}") @@ -42,26 +59,6 @@ for board in tqdm(board_ids): value = json.loads(col["value"]) no_of_files = len(value.get('files', [])) data.update({reversed_col_id_map[col.get("id")]: no_of_files}) - # elif "no show" in reversed_col_id_map[col.get("id")]: - # def extract_number_from_text(text): - # number_str = '' - - # for char in text: - # if char.isnumeric(): - # number_str += char - # elif number_str: - # break # stop once a number sequence ends - - # return int(number_str) if number_str else None - # text = col.get("text") - # if text is None: - # data.update({ - # reversed_col_id_map[col.get("id")]: col.get("text") - # }) - # else: - # data.update({ - # reversed_col_id_map[col.get("id")]: extract_number_from_text(text) - # }) else: data.update({ reversed_col_id_map[col.get("id")]: col.get("text") @@ -74,110 +71,143 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() if column_name in col_id_map: _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name - return _ - else: - print(f"failed to find {column_name}") + if job_name: + _["job_type"] = job_name + + + return _ # RA ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") -if ra is not None: +if not ra.empty: filtered_dfs.append(ra) att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") -if att is not None: +if not att.empty: filtered_dfs.append(att) -modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "V3 Coordination") -if modeling is not None: +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") +if not modeling.empty: filtered_dfs.append(modeling) -v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "V1 Coordination") -if v1 is not None: +try: + # Only needed for one board in wave 3 + full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") + if not full_cost.empty: + filtered_dfs(full_cost) +except Exception as e: + print(e) + +v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") +if not v1.empty: filtered_dfs.append(v1) -v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "V2 Coordination") -if v2 is not None: +v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") +if not v2.empty: filtered_dfs.append(v2) -v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "V3 Coordination") -if v3 is not None: +v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") +if not v3.empty: filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") +if not cors2.empty: + filtered_dfs.append(cors2) -design1 = get_df(df, "design invoicing status", ["to invoice"], "Design") -if design1 is not None: +# Design archetype complex +design = get_df(df, "design invoicing status", ["to invoice"]) +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") +if not design1.empty : + filtered_dfs.append(design1) + +# Design archetype simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive complex +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") +if not design1.empty: filtered_dfs.append(design1) # Design Revision revision_letter = ['a', 'b', 'c', 'd'] for letter in revision_letter: design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") - if design is not None: + if not design.empty: filtered_dfs.append(design) # Lodgement Phase 1 lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") -if lodg1 is not None: +if not lodg1.empty: filtered_dfs(lodg1) # Full Lodgement Phase -lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full Lodgement") -if lodg2 is not None: +lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") +if not lodg2.empty: filtered_dfs.append(lodg2) # POST EPC -post_epc = get_df(df, "post epc & eval. invoicing status", ["to invoice"], "POST EPC") -if post_epc is not None: +post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") +if not post_epc.empty: filtered_dfs.append(post_epc) -# # POST EPR, not right now -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") +if not post_epr.empty: + filtered_dfs.append(post_epr) +# post att post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") -if post_att is not None: +if not post_att.empty: filtered_dfs.append(post_epc) # Retrofit Evaluation -rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "RC stage 2") -if rc is not None: +rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") +if not rc.empty: filtered_dfs.append(rc) # RA NO Show ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") -if ra_ns is not None: +if not ra_ns.empty: filtered_dfs.append(ra_ns) # ATT NO Show att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") -if att_ns is not None: +if not att_ns.empty: filtered_dfs.append(att_ns) # Post visit no show -epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "EPC NO SHOW") -if epc_ns is not None: +epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") +if not epc_ns.empty: filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'WCHG Walkups-Operations {timestamp}.xlsx', index=False) From 233a64f03c050cf76618c594c6ad233958c7c2e5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 31 Jul 2025 14:29:03 +0000 Subject: [PATCH 93/93] submit this code --- etl/hubSpotClient/hubspot.py | 9 +-- etl/hubSpotClient/types.py | 22 +++--- etl/hubspot_surveyed_needs_sign_off.py | 4 +- etl/month_end_automation_wave_2_layout.py | 14 ++-- etl/month_end_automation_wave_2_no_12.py | 17 +++-- etl/month_end_automation_wave_2_no_14.py | 58 ++++++++++------ etl/month_end_automation_wave_2_no_15.py | 45 +++++++----- etl/month_end_automation_wave_2_no_3.py | 15 ++-- etl/month_end_automation_wave_2_no_4.py | 34 +++++---- etl/month_end_automation_wave_2_no_7.py | 85 +++++++++++++++++------ etl/month_end_automation_wave_2_no_8.py | 27 ++++--- etl/month_end_automation_wave_3_layout.py | 64 +++++++++++++++-- 12 files changed, 269 insertions(+), 125 deletions(-) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 269862c..27e6903 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -18,7 +18,7 @@ class DealStage(Enum): SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915" CUSTOMER_CONTACTED = "888730834" SURVEYED_COMPLETED_SIGNED_OFF = "1617223916" - NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000" + FILES_MISSING_FROM_ASSESSOR = "1887736000" class HubSpotClient(): def __init__(self): @@ -226,9 +226,6 @@ class HubSpotClient(): domna_id = domna_id, uprn = uprn, )) - if i > 1: - # Break after just for speed - break except Exception as e: def format_error_note(e): note_text = "⚠️ Automated Verification Failed:

" @@ -266,10 +263,8 @@ class HubSpotClient(): self.add_note_to_deal(deal_id, format_error_note(e)) else: self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True) - - self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'") - self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value) + self.move_deals_to_different_stage([deal_id], DealStage.FILES_MISSING_FROM_ASSESSOR.value) return all_deals def print_all_pipeline_ids(self): diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index c042d6d..35afaa0 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -60,14 +60,20 @@ class SubmissionInfoFromDeal(BaseModel): raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}") try: - # Check if sharepoint link is reachable and has any contents - files = sp.get_folders_in_path(path) - if "value" in files and len(files["value"]) > 0: - pass - else: - raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}") + try: + files = sp.get_folders_in_path(path) + if files.get("value"): + pass + except Exception as e: + print("Trying SGEC") + sp = SharePointScraper(SharePointInstaller.SGEC) + files = sp.get_folders_in_path(path) + if files.get("value"): + pass + else: + raise ValueError(f"[SharePoint Folder Empty] Folder has no contents after multiple attempts: {self.submission_folder_path}") except Exception as e: - raise ValueError(str(e)) + raise ValueError(f"[Folder Access Error] {str(e)}") # download files in url and check files are there: try: @@ -80,7 +86,7 @@ class SubmissionInfoFromDeal(BaseModel): if sdp.condition_report is None: missing_items.append("Condition Report") - if sdp.epr_summary_information is None: + if sdp.epr_with_data is None: missing_items.append("EPR Energy report with data") if sdp.rd_sap_xml is None: diff --git a/etl/hubspot_surveyed_needs_sign_off.py b/etl/hubspot_surveyed_needs_sign_off.py index 7794d07..6b5f48c 100644 --- a/etl/hubspot_surveyed_needs_sign_off.py +++ b/etl/hubspot_surveyed_needs_sign_off.py @@ -10,6 +10,8 @@ os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" +os.environ["SGEC_SERVICE_SHAREPOINT_ID"] = "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9" + from etl.hubSpotClient.hubspot import DealStage, HubSpotClient # Local development @@ -18,7 +20,7 @@ os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/post hubspotClient = HubSpotClient() # files missing from assessor column -deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR) +deals = hubspotClient.get_deals_from_deal_stage(DealStage.FILES_MISSING_FROM_ASSESSOR) for deal in deals: diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 5d208e2..7dca011 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -17,13 +17,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ 207.65, 101, 186.4, 98, 98, 450, 150, 163, 135, 120, - "Mariaane Please tell me", 45, 90.5, 40, - 25, 25, 25 + "60 - Needs to be verified (Post EPR)", 45, 90.5, 40, + 25, 25, 25, "check with Kevin" ] } @@ -124,7 +124,7 @@ filtered_dfs.append(v3) cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ].copy() -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design type archietype @@ -143,9 +143,9 @@ filtered_dfs.append(design1) # Design stage revisions design2 = df[ - df["revision 2 design invoice"].str.lower().isin(["to invoice"]) + df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() -design2["job_type"] = "Design Repetitive" +design2["job_type"] = "Design Revision" filtered_dfs.append(design2) # Lodgement Phase 1 @@ -164,7 +164,7 @@ filtered_dfs.append(lodg2) # POST EPC post_epc = df[ - df["post-epc status"].str.lower().isin(["epc pics uploaded"]) + df["post-epc status"].str.lower().isin(["epc files uploaded"]) ].copy() post_epc["job_type"] = "Post EPC" filtered_dfs.append(post_epc) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 962eb46..7fa4c1a 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -11,18 +11,20 @@ monday = MondayClient(monday_key) # Shropshire Council HUG2 board_ids = ["4718185486"] +empty = "nothing on rate card" + rate_card_data = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Sign off" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + "(185) - Kevin Check as this depends on property size", 115, 200, 125, 125, + empty, "(135)- Mariane said 'Remaining RC & design sign off", 185, 135, + 120, "(60) Post EPR, please verify", 65, 115, 60, + 50, 50, 50, "(60) - Please add price for design sign off, check with kevin and marianne" ] } @@ -117,7 +119,7 @@ filtered_dfs.append(v2) # Coordination stage 2 Please complete cors2 = df[ df["rc stage 2 invoice"].str.lower().isin(["to invoice"]) -] +].copy() cors2["joby_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) @@ -134,6 +136,9 @@ filtered_dfs.append(cors2) # ], "Design Revision") # filtered_dfs.append(design2) +# Design sign off +design_sign_pff = get_df(df, "design payment step 3", ["ready to invoice"], "design sign off") + # Lodgement Phase 1 lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") filtered_dfs.append(lodg1) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 9203f2b..4a644ee 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -11,6 +11,24 @@ monday = MondayClient(monday_key) # Decent Homes Stonewater - Operations board_ids = ["9319118237"] +empty = "Rate card info missing" +rate_card_data_example = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", + "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, + 260, 126, 281, 126, + 262, 127, 282, 127, + ] +} + + +rate_card_df = pd.DataFrame(rate_card_data_example) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -79,25 +97,25 @@ filtered_dfs.append(ra) # ATT -# att = get_df(df, "att", ["completed"], "ATT") -# filtered_dfs.append(att) +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) -# V2 Coordination -v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") -filtered_dfs.append(v2) +# # V2 Coordination +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +# filtered_dfs.append(v2) -# # V3 Coordination -v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") -filtered_dfs.append(v3) +# # # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# filtered_dfs.append(v3) -v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") -filtered_dfs.append(v3) +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) # Coordination stage 2 Please complete # cors2 = df[ @@ -107,8 +125,8 @@ filtered_dfs.append(v3) # filtered_dfs.append(cors2) # Design stage 1 -design1 = get_df(df, "design invoice status", ["to invoice"], "Design") -filtered_dfs.append(design1) +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) # Design revision # design2 = get_df(df, "design revision invoice", [ @@ -125,7 +143,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -133,12 +151,12 @@ post_epc = get_df(df, "post epc", ["completed & uploaded"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST epr" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att", ["completed & uploaded"], "POST ATT") diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index 3686198..8644db9 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -11,18 +11,21 @@ monday = MondayClient(monday_key) # WCHG SHDF 2.1 Mansard board_ids = ["5636990610"] + +empty = "Rate card was empty" + rate_card_data = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + "259 (new RA rate for PAS2035:2023 - old rates for other works - to discuss with KN)", 40, 178.5, empty, empty, + empty, 180, 275, 135, + 120, "60 - please verify with Marianne", 45, 45, 40, + 25, 25, 25, "Please check price for design revision with Andreas" ] } @@ -84,11 +87,12 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name + if job_name: + _["job_type"] = job_name return _ # RA @@ -107,8 +111,8 @@ v1 = get_df(df, "coordination status (ioe mtp)".lower(), [ filtered_dfs.append(v1) # V2 Coordination -# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") -# filtered_dfs.append(v2) +v2 = get_df(df, "v2 coordination status (ioe mtp)", ["rc complete"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) # # V3 Coordination # v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") @@ -125,18 +129,21 @@ cors2 = df[ cors2["joby_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) -# Design stage 1 -# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") -# filtered_dfs.append(design1) +# Design stage +design = get_df(df, "design invoice status", ["to invoice"]) + +# Design archeytpe +de = get_df(design, "prop type for invoicing", ["archetype"], "Design Archetype") +filtered_dfs.append(de) +# Design repetitive +de = get_df(design, "prop type for invoicing", ["repetitive"], "Design Repetitive") +filtered_dfs.append(de) # Design revision -# design2 = get_df(df, "design revision invoice", [ -# "Rev. A to invoice".lower(), -# "Rev. B to invoice".lower(), -# "Rev. C to invoice".lower(), -# "Rev. D to invoice".lower(), -# ], "Design Revision") -# filtered_dfs.append(design2) +design2 = get_df(df, "design revision invoice status", [ + "to invoice".lower(), +], "Design Revision") +filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = get_df(df, "tm phase 1 invoice satus (lodgment)".lower(), ["to invoice"], "Lodgement Phase 1") diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 6e497c5..b349a7a 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -17,13 +17,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 259, 101, 210, 95, 95, + 450, 150, 195, 135, + 120, "(60)) - please confirm with Marianne, EPR", 45, 90.5, 42.4, + 25, 25, 25, "Please ask for Design Revision" ] } @@ -143,6 +143,11 @@ design1["job_type"] = "Design repetitive" filtered_dfs.append(design1) # Design Revision +design_revision = df[ + df["design revision invoice status"].str.lower().isin(["to invoice"]) +].copy() +design_revision["job_type"] = "Design repetitive" +filtered_dfs.append(design_revision) # Lodgement Phase 1 lodg1 = df[ diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 1c17c7e..30b949f 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -17,13 +17,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 165.75, 72.25, 174.25, 174.25, 174.25, + 175, 175, 124.25, 135, + 120, "(60) - please check with Marianne", 45, 63.75, 34, + 25, 25, 25, "Please ask marianne or kev for design revision" ] } @@ -107,7 +107,12 @@ v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination -_ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +_ = df[df["mtp v2 status"].str.lower().isin(['rc v2 complete'])].copy() +_["job_type"] = "Coordination Stage 1 v2 remodel" +filtered_dfs.append(_) + +# V2 Coordination +_ = df[df["mtp v2 invoiced"].str.lower().isin(['needs to be invoiced'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(_) @@ -116,6 +121,11 @@ v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" filtered_dfs.append(_) +# V3 Coordination +v3 = df[df["v3 invoice status"].str.lower().isin(['to be invoiced'])].copy() +v3["job_type"] = "Coordination Stage 1 v3 remodel" +filtered_dfs.append(_) + # Coordination stage 2 Please complete cors2 = df[ df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) @@ -139,12 +149,12 @@ design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy( design1["job_type"] = "Design Repetitive" filtered_dfs.append(design1) -# Design revision -# design2 = df[ -# df[""].str.lower().isin(["to invoice"]) -# ].copy() -# design2["job_type"] = "Design Revision" -# filtered_dfs.append(design2) +# Design Revision +design_revision = df[ + df["design revision invoice status"].str.lower().isin(["to invoice"]) +].copy() +design_revision["job_type"] = "Design repetitive" +filtered_dfs.append(design_revision) # Lodgement Phase 1 lodg1 = df[ diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 0cc1749..214c233 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -12,7 +12,7 @@ monday = MondayClient(monday_key) #Home Group Wave 2SP+ board_ids = ["4254419092"] -rate_card_data = { +rate_card_data_sp_plus = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", @@ -20,14 +20,48 @@ rate_card_data = { "RA no show", "ATT no show", "post EPC no show" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 170, 70, 200, "check with Kevin", "check with Kevin", + 470, 155, 165, 135, + 120, "60 but check with Kevin as EPR", 45, 70, 40, + 30, 30, 30 ] } -rate_card_df = pd.DataFrame(rate_card_data) +emp_msg = "was empty in rate card - ask Marianne/Kevin" +rate_card_data_net_zero = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 170, 70, 200, emp_msg, emp_msg, + 325, 140, 165, 135, + 120, "60 but check with Kevin as EPR", 45, 70, 40, + 30, 30, 30 + ] +} + +error_message = "Unsure which client this one is - sorry!" +rate_card_data_error_msg= { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + error_message, error_message, error_message, error_message, error_message, + error_message, error_message, error_message, error_message, + error_message, error_message, error_message, error_message, error_message, + error_message, error_message, error_message + ] +} + +rate_card_df_sp_plus = pd.DataFrame(rate_card_data_sp_plus) +rate_card_df_net_zero = pd.DataFrame(rate_card_data_net_zero) +rate_card_df_error_message = pd.DataFrame(rate_card_data_error_msg) for board in tqdm(board_ids): @@ -113,9 +147,9 @@ filtered_dfs.append(v1) v2 = get_df(df, "v2 ioe mtp", ["completed"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) -# # V3 Coordination -# v3 = get_df(df, "") -# # filtered_dfs.append(v3) +# V3 Coordination +v3 = get_df(df, "v3 rc status", ["rc completed"], "Coordination Stage 1 v3 remodel") +filtered_dfs.append(v3) # Coordination stage 2 Please complete cors2 = df[ @@ -135,13 +169,10 @@ design1 = get_df(design1, "design type for invoicing", ["repetitive"], "Design R filtered_dfs.append(design1) # Design revision -# design2 = get_df(df, "design revision invoice", [ -# "Rev. A to invoice".lower(), -# "Rev. B to invoice".lower(), -# "Rev. C to invoice".lower(), -# "Rev. D to invoice".lower(), -# ], "Design Revision") -# filtered_dfs.append(design2) +design2 = get_df(df, "design revision invoice status", [ + "to invoice" +], "Design Revision") +filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement phase 1") @@ -164,7 +195,7 @@ post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT -post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT") +post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT") filtered_dfs.append(post_att) @@ -201,12 +232,26 @@ filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) final_df["job_type"] = final_df["job_type"].str.lower() -rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() +rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower() +rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower() +rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower() # Now perform the merge -combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)] +sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)] +other_df = final_df[~final_df.index.isin(net_zero_df.index) & ~final_df.index.isin(sp_plus_df.index)] + +combined_with_rates_net_zero_df = net_zero_df.merge(rate_card_df_net_zero, on="job_type", how="left") +combined_with_rates_sp_plus = sp_plus_df.merge(rate_card_df_sp_plus, on="job_type", how="left") +combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_message, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) +combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) +combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False) +combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False) + + +# TO DO check everything in excel +# make logic for seperation \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index d8f0a35..0aff6fc 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -16,13 +16,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60, + 45, 45, 45, "Design Revision check with Kevin/Marianne" ] } @@ -139,13 +139,10 @@ design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repet filtered_dfs.append(design1) # Design revision -# design2 = get_df(df, "design revision invoice", [ -# "Rev. A to invoice".lower(), -# "Rev. B to invoice".lower(), -# "Rev. C to invoice".lower(), -# "Rev. D to invoice".lower(), -# ], "Design Revision") -# filtered_dfs.append(design2) +design2 = get_df(df, "design revision invoice status", [ + "to invoice" +], "Design Revision") +filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") @@ -199,11 +196,11 @@ filtered_dfs.append(att_ns) # Post visit no show -# epc_ns = df[ -# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) -# ].copy() -# epc_ns["job_type"] = "post EPC NO SHOW" -# filtered_dfs.append(epc_ns) +epc_ns = df[ + df["post works no show evidence"].fillna(-9999) != df["post works no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 7e273f2..673ea66 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -10,13 +10,15 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) board_ids = [ - # "9349630181", # WCHG Walkups-Operations +# "9349630181", # WCHG Walkups-Operations # "8829428746", # 2502 Accent Housing - "8830772914", # "L&Q London" + # "8830772914", # "L&Q London" # "9601691730", # Cardo Wales & West - Wave 3 + "9660895490", # Northumberland County SHDF Wave 3 ] -rate_card_data = { +empty = "Rate card info missing" +rate_card_data_example = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", @@ -31,7 +33,59 @@ rate_card_data = { ] } -rate_card_df = pd.DataFrame(rate_card_data) + +rate_card_data_2502_accent_housing = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + +rate_card_data_l_and_q_london = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + + +rate_card_data_northhumberland_country_shdf_wave_3 = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + + + +# rate_card_df = pd.DataFrame(rate_card_data_example) +# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) +# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) +rate_card = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) for board in tqdm(board_ids): @@ -210,4 +264,4 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG Walkups-Operations {timestamp}.xlsx', index=False) +combined_with_rates[attribute].to_excel(f'L&Q London {timestamp}.xlsx', index=False)