From 3bd76ad8bbca88f0e1de5bbdbbef83b27bbee095 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 7 Apr 2025 13:29:35 +0000 Subject: [PATCH] save osmosis automation for khalim --- etl/osmosis_data/asset_list.xlsx | Bin 0 -> 22573 bytes ...osmosis_monday_to_sharepoint_automation.py | 71 +++++++++++++++++- etl/scraper/scraper.py | 20 +++-- 3 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 etl/osmosis_data/asset_list.xlsx diff --git a/etl/osmosis_data/asset_list.xlsx b/etl/osmosis_data/asset_list.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..965de921bacdb372bf04ccc09def9b1b72fc1851 GIT binary patch literal 22573 zcmeIaby#0Hw>C_1cPTC{?(XhT+={!qI}~W4xVyW%ySo>6FBEqu`u$)!XJ-1$dEf8+ z{T{9h*gHGx&dOSMlC_iU$V!5Mq5^>fK>`5*5ds}&`Sypr0Rl<{2LeI?f&^9?a=1t!Y?0tV#&-=F`7El?9HCDTm@>whBl5I~_)V}y+?V1f(5 z6ZQ#IOha(1bn-lR-tTB_D-VQ_iA87YTY=O=o6I2_ZYpgGwH84pR2#Bi6)p6s<3yhf z<=L&x3MU#^)VbQidM2_YpDhm$*IRC?cRHPuc$AZfy^Q3QRvqM2f}rD!(Ge*`UQ!B} zO56$x5#3(TLbIJHF8gxDgnikP2?dS0n`WKqoKc|Y!nTvt7OfPSdNp~J2CJwX`^r~gBjw2z1Fz6CNtcSA~uHKob^3!7_^f7VcmKW3D zsWM~@z`y5b5FpuqG-4F3HbY=QBVGfvQ5ZlY*0wXWu&1SY`TM_(_y6G*`?so>L`g}4 z&?EYv_&yVkv~n&{5Ph?t72cCFczi>j_^8907?IQTcq78^kXeW4mQwDV=9+kARF&(D)sJ_sJE+GG0OiTR|wya@4^B#_ka zKM&c!YFcuH?NPj!6Gc3B=~@^N@;_S94rblZS90r*sf@ECkF8eS{cDqfDGBpb0|5dm z1q_1-fE+Fsw9eLc=6crF<}YJjiIT2$G6(A8Q_dsA_6h0-g3JO3Um&$yr=qG7&u9uM z8n#{voixiN#i^(JGQynEb0));te(EM@3+hDQxr&<;$+$>jSCljq z1p4N>R2Wc_ksF8?7EumY9u+E~%x1hHMC{i&oeA1u+Hh5w5`iP|RTg^VsamBRX5#1t zt0C2+KbOQ_uKuaP7)@cEn~Y(ftHH&jf#3$Mp|vxu66rLx3t@?lZA(grqoK&CaXqSh z8j4RHn3>8xbb! z{!#v2V@l=8Wd0^_)1~m7*~FpQMhrCA^Xl)@WV&ahdVuH8q z8MnTH8_yi?45iJA7J${{PRwIhaJd$NKUjN%?!r}pmF9hoHxQ^3Ghd4g&ZYf69`U=( zHk2=XA$1+I>l?YNcCYTX-TZmm(Lj9HJnY6u%zH))X-^E}z<8JF#7Eo~#3_Q^9EBa0 zNGwJn8iS(_2Vs_MuqFQfp!oatg{APq-E{xSYfbke^;x)V^Pf?6txPQSCFHjpLL_yW zJKQIGXQ(n9QinV{V>%JkaLG67p zs}uIx{#u@pqikN2z`bKezeRKNk_H_1QqGZ*PKB7ma=rEQ{yOPtTb3x^m{k}T+p|k; z3&;7PVbgW8VMT4A*ju$*=$; zmjNgc5Ec;R8$dJsWAOTWWBhX{?D+LLRm@oaykW$YR3KX*zSN8OymnLhxn;%kAkH)EB9z&eRBi+l%P_c4(YJk< z)){+@OI%@K77MjSbliF2X!IIQU3?>|JbA1-L$76>S?&gOT`5%UZotVzAh$+Z5<0Uu zBYK~J@-tJ|+Yfr573=JLZX^wd?Q!r)r8QcaN-_#_N&|}n(assJd$tz!R7W_!^YfcczOzqvp6}GXb4+dFr0wgE z79%T~goVgsVaK^+kg3o6Lo37Lp)YuH6lexIH&Qnk+5La)(h`}Hr(D&+>%Xt&zc9SB zXED}D5sUw+{Y5-~q*J;NI8AP0>3uFMsj`#5)GB0hs?M2nV@tFW|1$~g9Hb>yz&01o z+|%SvR@BFh1Tx=dUGsJ&$TcTSRZ*0 zzEK>?9UikiP7eSb#KBNeNrNqU`EYmu=h8_#1&>uWOPbfy54}4_M%{LIL+ufc;AA^W zC%hUbU}#R=X;(oo$&z)Fzq@-ZaGSvowGLC1y^#KL;`EHUod-qc%`s+0#Gi5K^<`Lf(a~jxi>Wj6knOaO`eDYzr+h`b_!!?JzV(caT4!w$`f^wQt88x*S{25F$ z#CShYJbSEvo*JYwsf(NEPbb{RlY`T7nfrVhmg5z?$9u<{^M{**;fs2$vs@*d zydWluH0mBPy_MMaa~oGXA@3K6P0_B$<>WTs3rpJJdZinjMAbkO1xl4Lkj`z~h{Mx} z7ZXIygDFO71gw#=H@z?Q6|IjO~Q ztbS>%EIvPxEZ6;3kYhkaQgEYmP;mcT@0iL%MEVzRjF4{n7VsNUa9xs$kLp3sO? zn2gvwXkR}^XDW0~B4mAjc*OhxE3yshESbyO9WGkry^R1U>G--;@l7qk5h-+4js9k)ip3n&6k+taGn!Ci+j8CSA_lg7JJ#1! z$1lLWn5rq=q^c3o+xx2+3ndI<`>nv)-&V6Fz`c8E zbi=qgop-HpEqJeVYbAL$8(?Dv!Yt?Ty%qP%Yb=t`udCcy&L@)qFEA0g-F#hb6}1Kc zeBE1l=Z1jS+UhT6k2nc%l(q_&+l#$~PTowqpYFZ52dNc8=%n)n&{XIW53v;@eDy`? zNq0!E*o)U3R{`uA;P~tL?05Bzegm5vtDbo;c4A~D3Pu>%^m|qHD-{nxFhW{ytH1cW zs@qRgJZ|VWSVW_82DZ4~&u#24cO;<$n}BC~uVy6*4j18ZlK}mQigIX?|DHy;n{e}o`@aUD{62y`O6FVk-em^R%Qg!t}`AKe_24f}Z}T$1zns{{LoP5eqrkWV9g8yFpHWX$&`) z%`O1N^sHzA6n2U4`&Cixh~7Ub=fRJz{bxnNR6YKrCx*eWnnU&re=4R8}JCL$RCFAhW;G=&HfwY>NlDDhr?p$WnsG)c5P3pvn0np4Pr(Pw|M zxIl;~wbY$XG{gOpf+A9EuSQTPAD0ZjH3%TE_#&|QDv$#Za9s4cdle{s74Uu)IIMw* zfMZ!z?tLHirDb9O#E5ruyKv;VFO_?AbO@%VMa0Q0OwmZx2(QQ$fSJRaD(27R znRvc~U&#rcP}6@Vuf+46REIbaDV#3 z>O;HYr-haSW4{@p&Q_#}M>Xu@r$~y2+>TpGu;b9vv2-})U9ykHe%Eu05%vJX&^;p; zcn3N^TU2m}c5+}}w$J*fywu@M0^getEg&X-Gwjf_3mI?fSe9`LiFBYpbkqbmvYY@f z>ez+>W7GM-&;P;!Mj`Wwi+su}?RwS8#}wg4#D#VOyjX?0Asx<4$6Qd08GuWZaBahz zi~(->QeG>P4Y)eC9l%u(i#ov8bZ46xFX}b00v`kNk3|k0^5K72M7(*vfRWcc>EWHuv)Ooyc*@K?zwD7S%u7^s#NcRbL? zp`Wq=jyZYpWL`pt!iScESKB0b0?a^5nHN2Dcn4~gPkS3w3g(sCQ{;mf--3xlWsoYyqU94Jw$S{F{avK~MNn%h7lM84Jow_vu0t3qEOX$%5 z&@uqf*4Zh8fy4$=9s4}B6krZal-rQF$aXV;(R1m2rc5w@3IQ%l>3!z@xEu#?Zdv4| zMI_?xg+e{_^}vJycw7P&q~Fx>XrX0S(`lBq?#1E~Rtk6XqZ~b}>yXd!!#W3vmeZ^A zZmK(-Ei4Dv0peugP$~UAlHn{LQ?p+r!hbpYir(iw!R6;9c(Fa|@;LZ&Wi8>Zn4wb! zABhbC3N8N&H*8W<`TZ;`3)rhV)v~TZZ@h60PYLQN^inMW&1m2d7SG^?jSbd4O>@>) zKfRn*JCP_`AWQ*mL%_)Rl;DLuLPqL0I$`StXsdoBO0uYJrb{Td&MY|fUG5Q$&_VJ< zEv!+_HrvoAfby<9KaY&9^8+|~FJ@5Zh_EVZa|w9xy@*~yg<6NjC889LRkR2q5d?vO zlnp-kh5$e&9%!zCUup~;cKD5~^Q(zmpD<&Wi~+i;KQsV%5zhb|f#Cu#2f$y*xdlwF zffth5rdqB}Xz%5>3CUzl=c(hV`jQsSEx1L%CV&ioNltUrPYlgm|%^}U0~ zXWOa(*eL7Mg3Acu3BS>dpM}i&lP@ShOT9K>MFfV|ioL1~=xP6`{9Uv56+tPd*Nq=U zqmrC`PFXd;Tz(LVt_a^=`s4M>?iX`)drm3y%l*831^C8re)8^WRm=uj4;%q-6FBYY zw9apIN-*QocO#43n_tzq=loVf>4w2aDR}=D}q;L`n z`eN=E97Q9Q$ZwtiO-%ow_g{*v5p_-mzWU)Fz0f!7oEq6B0+UN@ct9J_7j_~Ay0h;=5B_hp zumwC48~)FtPYD=ZBGl1h0d2+R@{i(PivBN){dbpWqrVRiZ!BNw!BnQ_^r}^FIhKiPpWyR8Jscf{ueGa3UtV&B%8M_ycH4Q%V~7cFvLQ{T-7Q@)dkuDz z_gGOJ`wOr+E*%rWGEPxj!p(AN zHoI^sinz!wwm;DRwW!_qt#y{e6R%LQLt8x!g-b!i!ZR6*D3@2(>M~Xch96tem2Ayq z)|*N{$``4;*Qmb+EqlqM6upSjU_}`hvkxL(Zn*@{zDj(=UhchhfAT4dLih-3*<8Ic zr=UsJ9eB>NH!7=$?+8kx9!*l>$~xx&oXT_@Q=9e7I)x#OA$gTpGOoZ)8q;m8bjikc zox8Az)v$o|RlSoejrz=)4=Zt0a``YN{b^cy0;3Xg$>= zdJ0wW?_+4K15{f$7gpHt#cFP_DwubJ)0VX~c1M@G{#nA=G8i#WLsoKg-dptS^H>)IE>6MKuV-TYyxJl z8HHnPxu#z;x{__}<-Y=s$@W!B*IPYi>XeogK^qo8zkp9~B@9gxgn6{SGidnU}Q{^AM=4)R$@0S8{u~Spss30>xHddA^Fb$Y3rl zzKBn^Zh!k=Ww!UCi6f^Q=F;RVsQT#eycy4nD%{ffA(q=O%I~jQ{%B&hnO*^UwA!0T zfKRt_ym!W2YPT-8OmX#F=~??cJio~8s9pxBruoo(9pDlmo|G3tz3?i|W3r>kP^T{Q z;?VFIa(&7xK%r4h-l}evr<~H5?pbHt8Vh8Co0uU0yxNdHu+v zKO9%7c|BXoi&`PH0BMxV;!Kz>36iy$L=e5&|1k& zC$;Nsb4Qs4$#<;q$L`cy8qf*iS8s!S@3ra!vgSwtNew5C?qvIaUFB7Q-n7Qq!h>qD zUa{&f->CIrBFsw}LcK^ofCGA$*T*a%P6{{i0BnXa%}-X(Cv9^FD6QV!?)SG>2mNz% z&wzRHU-Mg4ZNXjqz_iFr7sF7E@_2e5 zDQ+_letsV>UTs$Ig=j=na5s>&)4zxx-T}V2m>40Cn>xyQBD0wDEA;}=F~eXrJ9sYT z0qLm5R|l3sa4b6fBfL=>cN55q%dveduJ~#GqvH4@4iRp>TYx0`?<=sMZIu z7%|FZEKqpRA`CS;?u(7l=c>g@P#s4e=z^%%>32)Uzf1ukTxk%)$S4nf<8J&IYa34A zW;4p&5xBEHN%7@ULX!dlsnR5kA%(2vJ(Qf-bKf|WE!T>1jG7a79%Co5HrXYnu;+(V zlEjHZG_fxegrS|Z<{11owT8)C`32*N2ZaIKT_Rmo9V7f+0ttqF_b$0Mhw|*5^pw~S zki`SS=N8mPhRb3UrqbEt z&3%b0dXsPh8^ zoio-2L1RQI0^VeFnj^U*O0>ihji2ixFtZFzxU z$t!)t&Q?S_`gJXeKEi__r0f%Z)?XgdZ zGY`m2y}ZMn)wlktY_7mKB;y?f^PR>$x&2jN^CJ7n%+!6z;1Ci}X|JY?Gt*pacz!w! zOfBnajc^wmF$`t$=d)PyQv*=zZdZr6`(s`-8`dkwe49dHnPk0cENHCK>wwc*X`(qh z5XqrLU*bLFSr^FQF+s)@r&h4b&2Q zk7qoKqp~u~e{( z2wy)=gl$cJXo1Tw8O~6$f7#MfwrMyW6Sm2`>xcYF#YxLin^*a49OZStbfu}9sjzkO zvBlkn(@*4LJ&f9qhK=`bV@DQIxS&P$$u2jo?LwsqBUyQvC*O5bYcAxH7u`!BC7NaI z!&mB(KTjcBH(&WJUlq)7_T*JC`B=k`4I3|c+`&fmBT6HwP;nj^K`keKW8g;Xt5@LB z-i`lY6i9N>NF!0L_>@3}n5ymAI-$h=J)qx}xdWLmISE5~UU@?GXu;R5dRVv`tQl== zXx{v)h`J?q9;e#si0+S$|noJvPISr?l*S%WQ1)e2HHH{ zcAFxq@2>GCoTXY5Y>xWOrDBE`FMDjsGOx0(@qunUSrc?b@=@Y|T~%PfQ~2plluUQ) zR)4daaf{`j$MNT$uSwoo-NkF7ThHLd3F{B7_=W>ic6@^stJTg@z_gU@#Re~Om5H$K zao^}$|2Ea$UcDoC!|_D)UkCc}4roYnz$TevC?FuXKL&bx2UiP2` zo_TM?yW_iD+c?mrTvIbL2mJ~7Yo$s<(GRpD%0W|=&u!7#$N_P6rv1oY9Wj1P+B>^B zgwbGiPJ+xZD=_puB?chHCJrT1=|61oI2yg3@gGRBAvO(huIa-V+?A2DbV^yZBx8mo zFu>gkrJw0?Jg^W1D;!r8CVo&LqyG9vA>K;URN{P*26-8d%X%NW>-{kzI*~ifSAx5_ zIy(kp3cdLxTO%iGbP*!;s_%TYXQb~1LwRT3m|a$gA9z-U>m(}IQjB1(Hx-e6 z6iNS<+H4Ayt+zx98m2kJjv-Ui=Dhf#X&=A7+dduk zNYqOGx-4Jr!@2dqhOsV2#Kz=rmgwtqclKE3yG69&;hb52ad?hl6rXhXK;Uujt7aRw z6cnSfwOyKbz0?(wumD_hUylH3RO$dvge!3XBD*U=Bn<-+I90MdpD%~@P!>yX6A?{j zoTC_Jqdui8%nLLhV&`T2<)NCb%L@8^JrUY>l5lzrsB0!z!Sxvg&X%!?Tm2e%cTi5S zq5STwy!kC}9cCh>Ug!9fSb5oExMV(t1e0tg_s@u>A{HE#tw?agC8?brMN7cUy#WGj zK0@t3elP(S%v^vi8dqgU3N*05l}B)}BYa{*1ftnOIVXBU+0Q!h6XMIAH>ftX9_8DJ z0kHs4isb{S1xn7oH4Dm}2`@LPjH@`??rlbB(2*8)lf~-Yx%s;JBF}ahG0uFj1v~rp zXU4vOGnJ(3(u5OZPh*6k8bo@Y5bL9vt@`-4mLryMp&)4{#i}9R+O+K>Ds!gmBP}~4 zi78VPi2=j0DAHwd#Y!AEz$g3T1?=7#%=IV81ajV01?)^VD-cToF3zXDE(Q*l>R&*P z=x4g-ZB-Sq&IdFsdmWnrt3Tzq$(7|MAM*!0u+O z$1ISG#Pd?OaK`hFDZc%=ad^Ht!cc8n1$S4yyj!S^hQ+k4b)T!u>KV=Pqw1MQcQp}) zN@_M4XFJ04&+y@@lDB=|ca%sAiyp04tQw_z!s6G0S_gDpskg&aZWXMWceJ3}$WaK3 zG-n;%TfLn7r&_vag;P~jQacbhO2($1r}aGU!m;mCHw+WuX^S^fPI?UMv9z3K*7BdJ zQD}PG#4h%$d*L)#h8PHoiHvPKS<``5+DEqF-J)bT`#vTG^7WNdiljzsH?8= zmfI$=5=*3Ya1OBguS1l#n$m1b%pbIjb(X93Ym)l!@+#B%%fQxR6WR7@9QNSms|t!g zbK4BEQma7G%-3!=gL)O>9$(VVD>RisC)(AOyid8T#=kw3MnCag$ye<&;sPv}g18)d zF{Sf*rgu&rZ}v`YUm)wIuR!Zv^Qh-FW3)-nvr!JsZhD4uZ?(4N5iKF&uV0lSE+3<& ztK?yKDho)oSy69a&XdIgSMpWZ`momJX2M$;lJw+Fr#A-63?VurJ7y)RaP;KzZgC;H zO4xBqf6n~MVfcl#sJ(H*@P=z81`^!&6DSD!Tsfr5^v2J~?mS4=fGh$+5Xv5yg_IM8 zL&cj$crAoAdz^wNP7N1`a%4S+?P}r?NSB_y0gG@Ou@h5q*Y#$;)*oCF!RXOdNFxz#C93e7)E;G9r*?-yme_w*{`}aZv4n$%82!_%E#m=w z!8LYZD>yX=l z$_F~`>yX2Som)1i09mT@D&diLoFrfQHpS9*--aB^yzA9ebnyz9lB5v*B1Tj-e-zlH zyT_(eNTiv(=bwFB3+7iv_`Ux_&eG?g_uRc7C1c0FmP0LY`^t4)6!jR-RB3_{W+Wor zCb@fg>=ZtD$gLN-3G!ir$8`QsK=*i4*`#N2=Q+uF9LZ_l0sCaJuR>uZ3PNdsI8R6> zZvm59iE%G5Bt;GLMh?e*r~mW)W#=9b;vE@k2goRp($UP4P@Xyk5z9^%X4a4tW0#U+ z6=oKN%eBH z#Je^)L?W%Ed>c1-^<(0X&(dh7ptb_5)p(^RmDp*e#3q!nS0$g)0@RD`i-3X!s@dt1 zBAt$J<#r4e1DnpN_SAtqv4IA8rRyC6(c$McaAeyi0=6NPAx)N~n!FTvq$h%X;1Jb- z#A?11E7Fs%6$nfyP>iZ)Gwi`Hs+cb8^dGQ8L4-nPwo(|hsad%a2fy2)j^GG-lnffi z^Ff}b&g`z9@8-)bf5^YgpxwJa{}xS?Yj33U!`tYPMvtgCfmR+Crk-gxkw^ z^I_d(_s#r3K}4mEMhh713k3yjr?w5`3Q$88L=aLEXy~H7s40-t1ubTKngLV%@bq!5 z4G4IBt)s5h+mjO!t^EOtKqGMgCGz;P3CARL5WdKe!LP00@V4V1L=iPu!Ijc>la*-!{2Xf17JL}e!%X9x7~O%=28$|lBH_w zM*~3`zw=G^<;JV6i|Hn5A5SAF)$|VUpBjt`#R#{R#J>VQ=^|^;u4oRS#7%KxnKK>_ z?j3btXwp5?bC@y<;I~mIMP-IjYAV8#`dKuatl0}EMhOibJ;R26X+Rxk&aq)A_GHiBQK^8ZAKA1!ABM)-4mEk2qS@0+g zu_TXJ_+E!{uTKV7SpN2IkpQS{qPn+d0W|+uL_)ID$Agq?ggdw7Gw80*=$|*gJE8Um z8~q;1E$GaR?Q*b)lVYk+6+389 z!L}r0pWuKX##zUdL@lplR`Sp_IwYBq>wZSj0p9?4=sSgk8{$hKToE>76r;=4144+e z8rV%Z=-H{omqJS3wKeM5%sNx=n);f2bVzdsb^J)A!UXeUlaM$3@W44v&wU|9>!Kg` zddMyZpSDq;=Ow5v}pn#r@0S;RS=KnYrO&7gcUW{0@~R(v1Ie|S-2i<1!( zy$eFUj+h}$yeFJ;_(^COQdvhMB9JZHzHgc%?0bHj&nWb`Qj~f+gw>Z<&EGIF-*g*0 zdHU~rZe={Hx?%wHg*br79tp7L*1%d{*3R0-o>t%5&hXal+>nUUN`7ET;ylHf3YA$9moECL`l*ndkf&^iemgwC!@{suRs;sb@ z6>^fTS*KU10=C3%asJBpQFHZs`$u#+HEMHX-Og*o1xLqg{W6b2jZXpVT%b&dj6@3` zDcW_gg|zMPX27v9NpmU`H5w6?pfKJGCHtes+F3$pfljelg$8BUjgCTlFlO#7^ca)QwOKpv3YGO{X-lv_ii&gCaa9PNz{6x{U9Emw!?6C6`88^}fO;0QPzM)449MFpI?~ zOCOJm%B@5*43Au1-PeFgLE5xR1W1v`V|vbjGCzaRq-;*CtnNTb`^%o&o+;~k8W0$7 zI}*IH^4ejg-16F|#dA%@{oTN8K#%sHZ(IXlODDQM{xk6w-S(LCb^=Hj_q6qe2DuPt7!<0iTnWJNr0J{*KR>r%6)wya19( zzLJa#NdJ{vA1y6tO&ts^U)RJ4vfcxcWf1&j?f>SgBud8Gj}ErCT_V;UlF0`cxTtVW zDX9j}NeVY#$QP9O$6i1FQ4?=~_>wN`@cI4EpM#^<4DiJ&mz``&QB>lC(9~>(hlRwu z1*YP4nv65i&M0>#bS3Ct;I5{lc=`&BsQALOVFyh*n%5mZbcBwWS^_O_NWrk^#Yl(@ zwvlazHG*D1CGD~&x^6!FpctMXemeWvm)GY{M!gNRFsVU+m5#;F|d<>YO{7=7xRY}#?0}MR~_BuawZ-;58UG zdk4t1*aU$@``BrH;yYi4b6~V-P1(j0`Vhv!%8>a+Z~nx{dx^%1BVOYJ*9N;TqU&3` zGydfD*ynt%|r%~{4(7lWSbGnT*FFoS#Mq&7) zNBsXs)m@eir?!qK2>u(@I0MNb!)eVNX?F~ZYHF4~y>@D{s36Blbz%zJiiT$| zLHY7H?f1m)DmZ126KdD60o@xTjN^MNGWTSPX(n+H;w{RfiJ25keNa%F0kvv$+UiBT zt=<6((4_R)^h<3ymv|>8pHW?I`_+_UA!nZJye?sa_S9O%ZIFBtZj}y}mXd4D4&hLq zu!esc*;_V;9gVRbhZ;7|!A#v&(3i^02AJFP@Wwor*3EJ~?E8ObgOMj;sU9TLQ{}E_7#i`Tk>na~I1wc|>3B0pezO>wjb=zgU z|A)k!{YIPzIvAVed56M?p3;bl( zS6xR7`M!UT@9k1QJ}9A%(8ma3S;+5EpZ&sS;e`?SYh*@DR>8GHy#_QR@4)53Mu_L?}a<$@z`FYo$P zyU1~F_U7vkrX2S)h<(KDHE1!ctKG7&{qHN8Z@D3Nu8hL>1av~3#`PdKSTmdSl zkK52ngc-iUaH&iXRKu)E?KKO&#_l;i=71$_TnCrmLg)VG5_K6rS{}XqB(0xBn3h)0 z-*eG-TYY~Ey73bs=7a?*F>k}(_X|{i+??wWmA>r5Gl5WVH3OY%O3=6~DmpqVYX zB!VPME`NFyRZ}Jpg@D+|8Z;_+M2vVe?Q_<)L>38(cA3wkLk8mjv8$m`P{7VU!F4oh zC2l#q%AHCKJ!C1|rx+@z(Mu=LXJfajwQ%P!)Z#@! zlJ!S=AA7|)NLJj~=aVKgK5?sMu+ae|XZjkM&G$uCz$aguoJ}UMV&i0R#BWYkEkRFB zgYgIye4oZ<(6!00oI`m%QNL5Wc)_5p%QSx`!0IQ!f*`~aB~-)qe~>JZ!*qZEBR1Iy z;F4)1>9@1+Hr2J{>HlONN<6C(qB)v?GG3Pnc8c?rrUI=>+(8C2o_;ykR2OKxt~w`4 zMw74iR7f4!*y`@-+f7_t1V8;MF($?tw}Em9~d3-H8T7F`C~4l~2u;|NE{{BkWpobj9E7Cgek*W+U!zF7$F3*uG;nWP9aIXgYH85aY7mf?;hGG zs-iSy^cuht7~N2c&jK1r9Ml%wiIDvY?@ z8KCQLb$FqUnoch#COqP~U{ITsDWjkyxRK3y6|CRqx_;iF#6NSwoli+RUFs&~`D)@A z2<&d<1SRX09=%;DhOKu`b#RMKn^n|R2)1Aq_sw!zxkpQ$?|EcHA7TMFC2VgnoRRyJ zYR;ge+_CjhD0wDGm`}ErPQK^57veCOHazI_yG!WSL_v{SmH=8e zy1w}4eLt)_l4Gf)!@Cl#c5()R94K>>rNfYF2*xlSFcII|L48A%F^&Av6%Bfj8*jIg zlIV7!zBkdYfEnmYwK z2d-1qDPz0J25RqNOl zqpLDAsyAkp&6A{O0qvvlLd0y(d22|!wt+kJP-;_k(z6Gq(GQG{$8fdsUDZk5cOYiP z+c>r~g7-&K%juIieL7wG@t6#Bw2=;%w2#AKw!zym1rX2jG$vA^Gr>27?m^Ua(-{cj z*9_;MWe~@9$Q~?X3@^#o9X)gE?$r8hyLtxl+~jR9W(0S7(vN5c40%iWP$8Cu8@rqo z1A_;$6Qgb=!qV1NqsU_;e|}i#qLoeZ*Z9s zcDIxYSy!{#!eKvtSR%<_R=4IrenIupu#n3@K;mf^c>0xx%heBH;*)CMd=&Q$sZX?u{c1RYjhJiQq;yHyA^h=Xou{0X77f>JY9tn3KyV^s<^&+o>)In9wrU|OOo}ov^82`cEJITFE zIdPTqK1R%sGB)>YSvl9mJUjZ!j&y&pcf*3luB$JInCtdC9f)g&pAGyfg&Pi`mTrZ? z5N~KoPGH_kxHN*ib!y~$^orV^N?wc2bda{H7# zDy03GiO@4P3DVoPGE_MrXH+7t9f4g5Jt2Rv3MWxU69b%X*KmzM6`%dbnd!y0oQ!hs zxLXq)&b5Os+tH6Mo8_r58!i1X!-NOZ!DG4B8>tzLA0354lYr38K(wks7%Q)Z zxvs$DLulVyrZ~0XTx!OkI?YC-Kh6*Do;?3+P#cwf;N9?l4r>2LK^!2c4eFvp6~6I( zMqK3~nVeOo56VSf4qs==xdMxVp zvEk973LGgqdsXU#iO*Z<{+-QGJopWmcalvu62F<3D_+XL(v;nPiP`9|w;`YVnd$OYB%J!*nNPy3 z^h)`s^K5{70w3-lyB&XDS#ABS_bvipl_kLMe@3;8|C6ZJB8(Fr5aolF`+ke2-kaED z@`0Ay0q$c9$}O2Yr@>#tgW&gxaZ5A8b9&$6h%+OKD{-FMiDHb!mAg z@T~dgNb!9!o;CVm_-;9^R{u48hi7Xk{=dA_{Euq;U+(XE5Oldun4eYaCHs)QuGMKN zrfr_-DwD(p`}NuC)&14HPXXc&8vyr%6z78bY&T+?(%Cep8G;azh^8hdOaRg2An*oz zVmPNt(vYN`MbUlb3Sp<6?N~zJ`9572TOz4BWHzN(Rag?YxTw@hy|kuR%1cm{bS}ko zD)ysMf#MDV93%cgXsBYBy&>j+i|qJ*GTZPOT1*^GZEmQ&if6eSbpP~U*VP?(+1hG?msgb_ zy?XlA#z7|QKCl5!t!7>}b@Fj8=YdLcMA!*J-?Yq~P{4=Lh(in+WZ{)15=B!>nir9- z>dJT>y_hpR8I|U+2Q=H?IJ)ITJ+w=8ujAqkQoqT5K+0U{64*fge)1tO*y7@=UI&c7 zvAKGzJAZws*M0fpzjg-h7r@#7HyXTlk^f7Bf2D!$D-9f8XfR9$paDcPmZv+o$_8c%cVI( zW+*Ot&rr|2Puoca-11_xKBi3Gf97penzA{qZ}% z@B2vq0>Gts1^8=M>F=VyZ@6uZwf0O-hRRz&~GE{ti$p{yV^{8vly&&k+CbD8mxJ zqrCR-zpL@j(E0BGHIlyrysGi9DF2K#|Bf>F;qNH_7J2@z(LW=WzhjL{|2x)S@yqX6 l{~Y%Ijx{U$?^u5gg0hlefPU#^Pbw-92VhWLk$<`S{{UQ8Z=V1F literal 0 HcmV?d00001 diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 70294d7..f9d2c25 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -1,16 +1,79 @@ import os +# osmsis keys +# os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +# os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +# os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" + +# warmfront keys os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" from etl.scraper.scraper import SharePointInstaller from etl.scraper.scraper import SharePointScraper +import pandas as pd -jjc = SharePointScraper(SharePointInstaller.JJC) +osmosis = SharePointScraper(SharePointInstaller.BAXTER_KELLY) +parent_folder = "Jun-te Kim Automation Test" +osmosis.create_file(parent_folder, "/") + +asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="2502 accent housing") + + +new_asset_list = [] + +# Create asset list and location +for index, address in asset_list.iterrows(): + webUrl = osmosis.create_file(address['Name'], parent_folder) + + first_folder = "1. Retrofit Assessment" + osmosis.create_file(first_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_file("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") + + second_folder = "2. RC Mid-Term Plan" + osmosis.create_file(second_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("SAP", parent_folder + f"/{address['Name']}/{second_folder}") + + third_folder = "3. Retrofit Design" + osmosis.create_file(third_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file('ss', parent_folder + f"/{address['Name']}/{third_folder}") + + fourth_folder = "4. Post EPC" + osmosis.create_file(fourth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") + + fifth_folder = "5. Trustmark Lodgement" + osmosis.create_file(fifth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_file("Doors", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Photos", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Doors") + osmosis.create_file("Survey", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Doors") + + osmosis.create_file("Loft Insulation", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Ventilation", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Windows", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Photos", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Windows") + osmosis.create_file("Survey", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Windows") + + osmosis.create_file("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_file("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + + asset_data = { + "Name": address['Name'], + "Sharepoint": f"Sharepoint Link - {webUrl}", + } + + new_asset_list.append(asset_data) + + +# Osmosist File strucutre + + +df = pd.DataFrame(new_asset_list) +df.to_csv("output.csv", index=False) -print(jjc.does_folder_exists_at("junte was here", "Tom Miller (JJ CRUMP)")) -data = jjc.create_file("junte was here", "Tom Miller (JJ CRUMP)") -data["webUrl"] \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 8651042..30917ff 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -24,8 +24,9 @@ class SharePointInstaller(Enum): SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None) JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", None) SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", None) - BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", None) + BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2") DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4") + OSMOSIS = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f") class SharePointScraper(): """ @@ -107,12 +108,12 @@ class SharePointScraper(): def does_folder_exists_at(self, file_name, file_path): - for folders in self.get_folders_in_path(file_path)['value']: - if file_name.upper() in folders["name"].upper(): - return True - + folders = self.get_folders_in_path(file_path) + if 'value' in folders: + for folder in folders['value']: + if file_name.upper() in folder["name"].upper(): + return True return False - def create_file(self, file_name, at_path="/"): @@ -122,8 +123,13 @@ class SharePointScraper(): client_secret=self.sharepoint_client_secret, site_id=self.sharepoint_drive.value, ) + if self.does_folder_exists_at(file_name, at_path) is False: - return sharepoint_client.create_folder(file_name, at_path) + return sharepoint_client.create_folder(file_name, at_path)['webUrl'] + else: + for folders in self.get_folders_in_path(at_path)['value']: + if file_name.upper() in folders["name"].upper(): + return folders["webUrl"]