From f0f9c14c03323142c557475133c3b49440233835 Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:26:46 +0000 Subject: [PATCH 01/10] Start draft PR From 0247e739ceadcfa44e65b30970c1f7f3c28979bb Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:27:18 +0000 Subject: [PATCH 02/10] Create comprehensive pytest configuration with global fixtures --- tests/conftest.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a75b848 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,51 @@ +""" +Pytest configuration and global fixtures for the Google Drive Search Application. + +This module provides shared fixtures, configuration, and utility functions +for testing across different test modules. +""" + +import os +import sys +import pytest + +# Add project root to Python path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +def pytest_configure(config): + """ + Pytest configuration hook for global settings. + + Configures project-wide test settings and markers. + """ + config.addinivalue_line( + "markers", + "unit: mark a test as a unit test for specific component." + ) + config.addinivalue_line( + "markers", + "integration: mark a test as an integration test." + ) + +@pytest.fixture(scope='session') +def project_root(): + """ + Fixture to provide the absolute path to the project root directory. + + Returns: + str: Absolute path to the project root. + """ + return os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + +@pytest.fixture(scope='function') +def temp_directory(tmp_path): + """ + Fixture to provide a temporary directory for test file operations. + + Args: + tmp_path: Built-in pytest fixture for temporary directory. + + Returns: + Path: A temporary directory path for each test function. + """ + return tmp_path \ No newline at end of file From 661cf994174f60a9edea2cf50f0ddec357120c61 Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:27:24 +0000 Subject: [PATCH 03/10] Add placeholder unit test for DownloadWorker --- tests/unit/test_download_worker.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/unit/test_download_worker.py diff --git a/tests/unit/test_download_worker.py b/tests/unit/test_download_worker.py new file mode 100644 index 0000000..0c85293 --- /dev/null +++ b/tests/unit/test_download_worker.py @@ -0,0 +1,18 @@ +""" +Unit tests for the DownloadWorker module. + +This module contains tests to validate the functionality of the DownloadWorker. +""" + +import pytest +from WorkerThreads.DownloadWorker import DownloadWorker + +@pytest.mark.unit +def test_download_worker_initialization(): + """ + Test the initialization of the DownloadWorker. + + Ensures that the DownloadWorker can be created without errors. + """ + worker = DownloadWorker() + assert worker is not None \ No newline at end of file From e8519e666a20e4b4d0787680a71aff6d29945ba7 Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:27:28 +0000 Subject: [PATCH 04/10] Add placeholder unit test for IndexerWorker --- tests/unit/test_indexer_worker.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/unit/test_indexer_worker.py diff --git a/tests/unit/test_indexer_worker.py b/tests/unit/test_indexer_worker.py new file mode 100644 index 0000000..cb065d1 --- /dev/null +++ b/tests/unit/test_indexer_worker.py @@ -0,0 +1,18 @@ +""" +Unit tests for the IndexerWorker module. + +This module contains tests to validate the functionality of the IndexerWorker. +""" + +import pytest +from WorkerThreads.IndexerWorker import IndexerWorker + +@pytest.mark.unit +def test_indexer_worker_initialization(): + """ + Test the initialization of the IndexerWorker. + + Ensures that the IndexerWorker can be created without errors. + """ + worker = IndexerWorker() + assert worker is not None \ No newline at end of file From 5abbd7bfa783636415722955977dbf561f5399fa Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:27:34 +0000 Subject: [PATCH 05/10] Add placeholder unit test for TextExtractWorker --- tests/unit/test_text_extract_worker.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/unit/test_text_extract_worker.py diff --git a/tests/unit/test_text_extract_worker.py b/tests/unit/test_text_extract_worker.py new file mode 100644 index 0000000..4d85bcc --- /dev/null +++ b/tests/unit/test_text_extract_worker.py @@ -0,0 +1,18 @@ +""" +Unit tests for the TextExtractWorker module. + +This module contains tests to validate the functionality of the TextExtractWorker. +""" + +import pytest +from WorkerThreads.TextExtractWorker import TextExtractWorker + +@pytest.mark.unit +def test_text_extract_worker_initialization(): + """ + Test the initialization of the TextExtractWorker. + + Ensures that the TextExtractWorker can be created without errors. + """ + worker = TextExtractWorker() + assert worker is not None \ No newline at end of file From 2a55ca77789f01f8ec4f8940f6c8715f945d7442 Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:28:05 +0000 Subject: [PATCH 06/10] Add __init__.py to tests directory to make it a Python package --- .../__pycache__/DownloadWorker.cpython-312.pyc | Bin 0 -> 3650 bytes .../__pycache__/IndexerWorker.cpython-312.pyc | Bin 0 -> 145 bytes .../TextExtractWorker.cpython-312.pyc | Bin 0 -> 2005 bytes tests/__init__.py | 5 +++++ .../conftest.cpython-312-pytest-8.3.5.pyc | Bin 0 -> 2381 bytes ..._download_worker.cpython-312-pytest-8.3.5.pyc | Bin 0 -> 1537 bytes ...t_indexer_worker.cpython-312-pytest-8.3.5.pyc | Bin 0 -> 1528 bytes ...t_extract_worker.cpython-312-pytest-8.3.5.pyc | Bin 0 -> 1563 bytes 8 files changed, 5 insertions(+) create mode 100644 WorkerThreads/__pycache__/DownloadWorker.cpython-312.pyc create mode 100644 WorkerThreads/__pycache__/IndexerWorker.cpython-312.pyc create mode 100644 WorkerThreads/__pycache__/TextExtractWorker.cpython-312.pyc create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/conftest.cpython-312-pytest-8.3.5.pyc create mode 100644 tests/unit/__pycache__/test_download_worker.cpython-312-pytest-8.3.5.pyc create mode 100644 tests/unit/__pycache__/test_indexer_worker.cpython-312-pytest-8.3.5.pyc create mode 100644 tests/unit/__pycache__/test_text_extract_worker.cpython-312-pytest-8.3.5.pyc diff --git a/WorkerThreads/__pycache__/DownloadWorker.cpython-312.pyc b/WorkerThreads/__pycache__/DownloadWorker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a0900d62e84dae9b972990aafe986b8aad3c5c1 GIT binary patch literal 3650 zcmai0U2GHC6~6Ol&%}1(VC;kd3FK$Nh)sZHH$uAt1R;UlZKM{e72Znj#4~Zmj6K7h zael1v3bjZz65HZ!t7KJG8!9A1L0Vp_@>q5Ep;aFm%aLL(S+!E7O1wotD@0#@#7;iS4D*WMLQk|4dMAGAS@yUDU6O1^_EF6Zl6uC z(6Tz0Wdq zTp;b<@*f%aii%T>{kpT#yX%1e^V7h7%`75^aZnQEIyo%#7_Vo0H;rxx4*E2Rki%)n z;qNd%XBf1A;>={6C*0M%N&+dYDr(NwWZfcyl{ZyP#AH&-X?8Lh_YvMw^=VAMDGmT5 zyo}Qp@m-m7TPc}L>9S=dla>#x`zT#|b#z2F%@M4chUN0)k&*g|F3n(7R;-b4pL5t; zz#$l2=xWh>+kpKeYY^DWYp529mH3-tZP#FlUlY4G#Lx|YMeJT0TovQXZ&bzjbMQl{ z=2bw)4}W@<&zC{Fh$Q$qC4qFaJ#@#WI^IxiL(Lbb;`KV z_j(N-Jy{EV#|ZkFTSyTUpTl_o_V#1F4d>`=afPn8H_s36wk`$78*pDW6))Qwh9SJ=@4XwZehX#EiwK)HR#Cd-AoYTbLj z5I+~c2!0-1`TkO{+C6eh*z}=j&(dJ|gVIDTwtH#3oGo2=;X^xPi=*Yy#gpZeclz%h zxP9R6;oFB-W*@4LE>!!+*CG>t8TUnf^T{_(1uu6#VR_Mx^BjxI4v@ul6%AFjrZ zu7!`)!aLW){grV4^61KmweZ`pQmFG6=Wm=}5B64qy=%dJn|o15)xKO8L}D*Xue|veb+7IFdnMC_D8T*kt_1HJFW{!?CO?$q;4VxBdussbLTbrh4Xeghj!{+2!FwE=LJanMBG#(dlADjm714cy4 z)HPe3w~06f8H_>^(^v^k6tOr?oq`b9O#|Diat<6h6RU)SAr`%KirG47@c?uS-7TPP zK9c=lwI+aFL(fjHP5xE{2s6+Nt^|5(k*}+hHCw(N^@?%~);I7~xW8rzdT-+eKA31F&8O zQ2btGF8XbnhFo&|j<2Z~6|pEek|O32KJAE^-JaK0>+RfqBW*>=-s{;E$)*T%$!W{b zFW*AmSf#B1AoA|C05RT^;dRwlXKtQ${27X^7V_qxB?)8-idvT-D@2Qdt=%|*tvN44 zIamyQ5?E-ob4)XH1SW}~U~N_R{YiN@VNzqzQ?|RRnO3$Iv2enA=dgNdwa_~X- zmwWH+z28@joG6WL#QGMEvay_Au^$bt#ZJ*ows$dG&i>L^kBwAfBM;|RVJ8$i$y8l6*lcKMzUO4Jc&JLU%MPI!UrhHTyi*jG1 zz*w03IxK)37ipzT18y=&+LFnvq2!?|NRnVOc{MNVb&Fft;zKZ-8(!U;CZR^%oKkfi z%4`PH$Y9Yj^K|`t(3%oc7N7=NWxc$PUa*{S=$Y6d?0?oT3DN)dv+*VZs1hCe4}y+6;r{_GWi(R& literal 0 HcmV?d00001 diff --git a/WorkerThreads/__pycache__/IndexerWorker.cpython-312.pyc b/WorkerThreads/__pycache__/IndexerWorker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..15c93f5b768237c698b23c8a08db94ffc9d85ac7 GIT binary patch literal 145 zcmX@j%ge<81UGLQWGJ#QFgylv(7|V*1jBTON(N0vzm*I{OhDdekeXix`iTVv`bDV) z`Nd!|-atP*zbHGkC?umOH8G`F-!m^IwIa0$BB)nTd5gm)H$SB`C)KWq6=(n>5Ep|O NADI~$8H<>KEC3r^B6t7* literal 0 HcmV?d00001 diff --git a/WorkerThreads/__pycache__/TextExtractWorker.cpython-312.pyc b/WorkerThreads/__pycache__/TextExtractWorker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d57b355daf147099836da94f5bf8a470459e5b1 GIT binary patch literal 2005 zcmahKOH3PAaNd6GHL(HXB!-YCM5YZJDUhOmXhS~=(9%@pl2lbzl~xU(Q1jucgS>z$eR z=Cd>NX2!oI5>3F?y?du7{t^NH-x_wyMaMNv+7cYTXFJx%9zReKjhVKebw9K46x&445f>Xd zE6c5fOQ;h&-HiB>MQQ<$P7~%}9GPfV(L__21x=cRNu#euwXk=mpMh^t6)LC#I-_fg)I$Q8mn!g;0#FfbsEFU8uMvVNP|+CshIqMT zQQ0qf7N&}>+eMq`x++u2x14dz?~O_Z&iOQQsT|;#u4f&?_jTRp4s@|~<;g33hUfKR z%X9qz>Zkf@4xgXEmSOsR&qUJeP2o7xpFZ>~D|92l`2JxA9$$gY)WJDvLAkFa=A&y$ z`%?EEMO}VrT~T)!AF_zU`eKm+kA0_=WvdD?_EaDra-YKay$m%DO>|3Gsau;tBo=b= zK?#Su8m^!{Nr7-@gH;hs(Uii1nh{8I$jP^cR0HQz4LYQrqj4aVYv6__p+}#x*k{BU ziKIiR=UD9s(O%}7NwqtomtjK@?~>NWT(%*Yiv-`S84smfjkgN9_y8tS0`yNMS< z-3DH%lL!|6$N-T-PCgP+4crY_L~wysbr~kb3s6QGco}?D7WgS*&%#@^b2T>hkY79R zQ(_k^w?wAp5*a@^KqGdp=wgdX!v-;EqTYCx$qRH^>LrteqO#gjYYH)E5h}Ch;q5b0 z@W>QZ2ovPJN~0cjvzG5u(I=P+u1_V;AQM#aJ;&x!RJq6|O3^5=ueqL8q*C6si&QRS z#-=f1_?L9kEm~>>^SYwSIGZ+s2`Nw)J?5v#4WV(L=YzLlR~JrlVG);#(+9ZMBTvtc zUUtk2(7(d~YpCY7Z9z<=H#?8s7`Q%gWAOUmk7#Ldy|e$Sva7)U!yBp0S}L_pCKEJ2Ov#=`&@`-;V&GIqa7V^~=cBR1}w2L{+ z8wPhVK${KE&N{YLB)t=ac$n8E=5h-ERzUrLF*R8w;=XAyz>B#K7bb)y#+1o Xw;O3g$ChFn?LBMlJ^ujP1UCNzM?}Vj literal 0 HcmV?d00001 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..2589817 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +""" +Test package for Google Drive Search Application. + +This file makes the tests directory a proper Python package. +""" \ No newline at end of file diff --git a/tests/__pycache__/conftest.cpython-312-pytest-8.3.5.pyc b/tests/__pycache__/conftest.cpython-312-pytest-8.3.5.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf65b13b688ff7e3dfc4a71e01090255f8a0b89b GIT binary patch literal 2381 zcmah~L1-IC6rEkIv@6B76x&S+re>1R5s2(pQlyFBc z=yJy>7;;V)lEkQvMh<&(_01k1akfJABX!S48ol`#&9RQi(9)_*e!Wp2KsgzXTT7Ua7fUv4gV&M*|vzZ~t>3Tj(29?cbFu@FI|R zy*D^kr~7USYHHc|vY)~*jzqBHx;G_c9T++B#U&t4VIs+g9moM3K&eWK{^eT$Rd}sP zU*b}?*iFJZwt?PRF2C1ss6UOx5?i5}#eoyza4F#99x2iiEdt*he4vb ztl5`{*9la(!>6TZk)GUV(|GPAU!eykpv@)%)#ZUEjtz8(LNZ#_rB0y3k&aa-fnk~@ zS|g^pjhMJCGk^pGo$yQ%9(w~saXiLR_?>nuY0CIV;Wy=(3*dlIyfwVs14VEi{f(B;laQMUeoNZ&YIz6a~9`>iJqQgkHa zi?Ja%Fo^Lc4-%n}i5?Lo;1KY{GO%E$1sN7(L=X{Upm+XQGM8lhOkB7x<&2r|r|~{} z8#Y8*@b7@MV~V2esc}X7cL*uU8u~W*0;OJ{!FB>E+Pbl7tfyDgo66?YR{lgbpO>5r{y$5uL~sBA literal 0 HcmV?d00001 diff --git a/tests/unit/__pycache__/test_download_worker.cpython-312-pytest-8.3.5.pyc b/tests/unit/__pycache__/test_download_worker.cpython-312-pytest-8.3.5.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31be7f144db709c8dd99b9b95812fd75fe2bfd2f GIT binary patch literal 1537 zcmZ`(TZsyGBrbvw$Pn;ubA1V=g{F^v+EVa~(2nGKHscv1$=TT0 zux$g$KS&Dw17&|oAS6voil9*FQ{Ngk5Xe)HB+uF|sjzh}IyZgiXr4Bk4Fv0o+3r8X z2>oS+as_kdC-;;4%_=0S$Lfuof{m;@LNs@)p)Cs8+olS)<}taaBg64Bmf8EirK z&Mf$n;~=h1rj4)jR1+nMa|<47o5`)c zddV$lne84@A1=jreO*{tZi)`)HOghn^;N4?%N#@gMb+=K#bs1p{VAB#5SmqZjYrc; zik%edlVygbNT*8Of=%z-s`{2vp z&$XY2?MFACY(78Heiho^Z(jDo@&3YUz4UUy{u;KQY#!SjGrxrGBS?herimyNolt7( zCH&Gp0CSG|hIKq#0B=W%eVJ+i|5#RP0hQQ3;x@+(}ywRs-8OgH{LfQ|fso(Zry-(#u5bb+bS7H07f-!y_ggAV=jxc_JerW!IZvBa)hv&aL X{|axsz#FfF2!DK3LBVOqxtRX~G;@r3 literal 0 HcmV?d00001 diff --git a/tests/unit/__pycache__/test_indexer_worker.cpython-312-pytest-8.3.5.pyc b/tests/unit/__pycache__/test_indexer_worker.cpython-312-pytest-8.3.5.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9736e94c87803da06a3036b520e5ad84740890b GIT binary patch literal 1528 zcmZ`(&5Ij16qm+hkJnjmy2<8?#YMCfG8gYQ+s!G2^pHX?Ic*8}BD5npp55_`k>u>! z*d=WPE%Xma3jGJl{vWyYk~A$Tfx;$U&z^r0D)TY`gJ?t>j*eO>-$YK`he%yoi0_6vx3b&>oy=;KiwjwZmxz>04RFQyDv?aBrGTUf+SJ^PB=MAb*~s|BBU&&YP$%4hqpC=SzhQEk1v04 zeK6XN74aHk#hqB0+L4219GD86+ihpO#*`9LJDXjw0p)JIB5hwyoj3m zMx_|Yr05Tr6lN8U>HaXyG$7j4vH~oWxk?q3mvLuGA7sTo%V^H}!mLomy23^_sl{XfJ553?WKUx%Sswz0z_L8?dTWm1c zf~19VJKf%LuXbw)BXTTF{kx^UJMF>9yytGctOv8jnaV4W!XjC5*dC$3*3jDKxAf5` zzg}xS4_n{de7f=cTwoV}rikl{)lotu5 zre5}5TI`!^)K9DJdj-91OK~Jq4V^zznP{eywsuqVE@snzwDx*{F|OK6`14o@`%E61 z(00vT{RbqkLIi2Qm!^Kpf9-C|_d&ETS>1r`&kDx)O%USn?^T5HWAs(?HQN0RM_+G# YvH1$GzrgEnf(XBRT0y}D$GMpQ0CAX!4*&oF literal 0 HcmV?d00001 diff --git a/tests/unit/__pycache__/test_text_extract_worker.cpython-312-pytest-8.3.5.pyc b/tests/unit/__pycache__/test_text_extract_worker.cpython-312-pytest-8.3.5.pyc new file mode 100644 index 0000000000000000000000000000000000000000..357070e6d07594465b8ff6ff4812be151f8e2658 GIT binary patch literal 1563 zcmaJ>&2JMq6t^dnOd6W9UCNhGz!E~cb4klr#erSz9=beAYHy$-t;A`c?M#!Z6;|T+;otl4d(ZQ-Uauiot9qsL z4@T&t5sKwanB5GRXDCK36yw-Sam%x`)b%3|qi(rX!R{nz1#nVItKGTQ97a6YTq>MR z>E?+kUvRPi2<=RS*8IQ=%fkn?n^~fW;!;V{&IM5&PS*LhTH98FMe0^A?r=f6dE8HV zvsPR0Byu8=D9;p2GC8qUIl0TyBxZ_RFYSI7sU*)prv@Z%AMUzY``2W2BgA^~K-{Rv z+|U62CHU`dKnb5A1-o0tmBvt>F&?22?&GmH^5hF;@OEwb1vjNJVA@bfW6}0@CWzQq zCX!v7)qhN)mz*>jdu)RxQQWEIqkeRlA zqoc;knyzKCFE|WHhdCDqI!q##kqu5F!C`J4x>M}+7b}wa_tzK9b&^e@!m@qEOOS8g7Cpg0^v%;z@gP|!g`Fd@c z^?J(?G?#X#E6esJFZZEUmZb{bG6c4dnYt1+djtK0z4{@Cbbpt{NeE_UTXNX$p)U() z;oO@#x%b<<(~VbtAYr9I3Z^QYg~g+M5* z>wwZYk0{mEq7A}42Ys5l#x-5ppv+Cdw?v|#341CXOmxB|uPe?(pL5@{^$v_%ESm?$ z(T#@RW#W$ZO%d(=|3KmtcoF84s;k%C$M9!y9z^rN Date: Fri, 23 May 2025 20:28:10 +0000 Subject: [PATCH 07/10] Add __init__.py to unit tests directory --- tests/unit/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/unit/__init__.py diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..ec13540 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1,5 @@ +""" +Unit tests package for Google Drive Search Application. + +This file makes the unit tests directory a proper Python package. +""" \ No newline at end of file From bd3d974a29573e858cce33bdc49b493b293be312 Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:28:16 +0000 Subject: [PATCH 08/10] Add __init__.py to integration tests directory --- tests/integration/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/integration/__init__.py diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..7807d3c --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,5 @@ +""" +Integration tests package for Google Drive Search Application. + +This file makes the integration tests directory a proper Python package. +""" \ No newline at end of file From e76d8fdcd36116ee834503f0d03d8e37ec71b9cb Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:28:23 +0000 Subject: [PATCH 09/10] Add __init__.py to mocks tests directory --- tests/mocks/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/mocks/__init__.py diff --git a/tests/mocks/__init__.py b/tests/mocks/__init__.py new file mode 100644 index 0000000..9fe93d1 --- /dev/null +++ b/tests/mocks/__init__.py @@ -0,0 +1,5 @@ +""" +Mocks package for Google Drive Search Application tests. + +This file makes the mocks directory a proper Python package. +""" \ No newline at end of file From bc349ab5dbd998a19459a2aef5e34d4b2378f862 Mon Sep 17 00:00:00 2001 From: relayrelayrelay Date: Fri, 23 May 2025 20:29:35 +0000 Subject: [PATCH 10/10] Move DownloadWorker to src/worker_threads directory --- src/worker_threads/download_worker.py | 59 ++++++++++++++++++ tests/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 262 bytes .../unit/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 278 bytes 3 files changed, 59 insertions(+) create mode 100644 src/worker_threads/download_worker.py create mode 100644 tests/__pycache__/__init__.cpython-312.pyc create mode 100644 tests/unit/__pycache__/__init__.cpython-312.pyc diff --git a/src/worker_threads/download_worker.py b/src/worker_threads/download_worker.py new file mode 100644 index 0000000..24da26f --- /dev/null +++ b/src/worker_threads/download_worker.py @@ -0,0 +1,59 @@ +import threading +import queue +import os +from apiclient.http import MediaIoBaseDownload, MediaFileUpload +from apiclient import discovery +from oauth2client.file import Storage +import io +import httplib2 + + +class DownloadWorker(threading.Thread): + def __init__(self, que, credentials, *args, **kwargs): + self.que = que + self.credentials = credentials + super().__init__(*args, **kwargs) + + def run(self): + while True: + try: + _file = self.que.get(timeout=3) # 3s timeout then close the thread timeout=3 + except queue.Empty: + return + file_id = _file["id"] + file_name = _file["name"] + + # To avoid downloading files that already exists + if not os.path.exists(os.path.join("Data", "raw", file_name)) and self.validFile(_file["name"]): + print(f"${file_name} downloading") + self.download_file(file_id, file_name) + + # Task done for notifying que.join() + self.que.task_done() + + def validFile(self, file_name): + supportedExt = [".csv", ".doc", ".docx", ".epub", ".eml", ".gif", ".jpg", ".jpeg", ".json", ".html", + ".htm", ".mp3", ".msg", ".odt", ".ogg", ".pdf", ".png", ".pptx", ".ps", ".rtf", ".tiff", ".tif", ".txt", + ".wav", ".xlsx", ".xls"] + pre, ext = os.path.splitext(os.path.basename(file_name)) + print(ext) + if ext in supportedExt: + return True + + return False + + def download_file(self, file_id, output_file): + try: + credentials = self.credentials + http = credentials.authorize(httplib2.Http()) + service = discovery.build("drive", "v3", http=http) + request = service.files().get_media(fileId=file_id) + fh = open(os.path.join("Data", "raw", output_file), "wb") + downloader = MediaIoBaseDownload(fh, request) + done = False + while done is False: + status, done = downloader.next_chunk() + print("Download %d%%." % int(status.progress() * 100)) + fh.close() + except Exception as e: + print(e) diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6124470ad852d122dd3975e6ae422d53cc7bb6fb GIT binary patch literal 262 zcmXv}L23gr478J_4Tk(d&{HnEc|a)?0=vvOr8W&4r7p>OFA_Sz4W z=GIfap_zj;1JX$IIh`tz@zl@VYwllzxIq4s(G*)&a0CEa8))l_$c5m#QqGk6wnFk3Gm zX6MT8V6YbT-h=wQ_c5EvD%WycxYvxIMI1}~k^MB2x2mm{)gb1R=NI|d`veJNiV%$5 zm^H$R7;cEx9l|yHCxm-6Y#m&M(0OZ;r@oQ$!TAU)pNF<)gAtNL*!>@CJ)=#;0~iQQ z0U7Rw;_Qm4QxiTRr#z67V^+FxYsd8Fr?K5cIgkhXsY^mLK Yc*&=Eu>_1I|JR8S;(Jnyn(Hio0gMPyUH||9 literal 0 HcmV?d00001