Error from CV2 point_coordinate_variance >= POINT_VARIANCE_THRESHOLD in function 'computeOmega'.

[xin-ran-w]

I meet this error when estimate an video generation model generated video clips.

return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
100%|███████████████████████████████████████████████████████████████████████████| 47/47 [00:20<00:00,  2.35it/s]
flow precomputed
100%|█████████████████████████████████████████████████████████████████████████| 280/280 [00:05<00:00, 53.14it/s]
propagate in video: 100%|███████████████████████████████████████████████████████| 61/61 [00:01<00:00, 33.36it/s]
propagate in video: 100%|███████████████████████████████████████████████████████| 61/61 [00:01<00:00, 34.64it/s]
 init edge (19*,20*) score=np.float64(4.8253655433654785)
 init edge (19,18*) score=np.float64(3.7121148109436035)
 init edge (18,17*) score=np.float64(3.5358328819274902)
 init edge (20,21*) score=np.float64(2.1367027759552)
 init edge (17,22*) score=np.float64(1.9133658409118652)
 init edge (23*,18) score=np.float64(1.511080265045166)
 init edge (17,24*) score=np.float64(1.510663628578186)
 init edge (20,25*) score=np.float64(1.464857816696167)
 init edge (17,26*) score=np.float64(1.2983629703521729)
 init edge (17,16*) score=np.float64(4.1183013916015625)
 init edge (16,15*) score=np.float64(3.7599825859069824)
 init edge (15,14*) score=np.float64(2.3618717193603516)
 init edge (35*,26) score=np.float64(1.4097154140472412)
 init edge (14,13*) score=np.float64(2.706547975540161)
 init edge (4*,13) score=np.float64(2.092278242111206)
 init edge (4,11*) score=np.float64(2.08540678024292)
 init edge (44*,35) score=np.float64(1.6958266496658325)
 init edge (3*,4) score=np.float64(2.4737095832824707)
 init edge (6*,11) score=np.float64(2.240061044692993)
 init edge (6,7*) score=np.float64(2.0445797443389893)
 init edge (7,8*) score=np.float64(2.0244669914245605)
 init edge (44,37*) score=np.float64(1.9792639017105103)
 init edge (44,39*) score=np.float64(1.911988377571106)
 init edge (46*,39) score=np.float64(1.9003345966339111)
 init edge (51*,44) score=np.float64(1.7495344877243042)
 init edge (53*,44) score=np.float64(1.7428754568099976)
 init edge (37,30*) score=np.float64(1.6048235893249512)
 init edge (37,32*) score=np.float64(1.5745713710784912)
 init edge (37,28*) score=np.float64(1.4908398389816284)
 init edge (3,0*) score=np.float64(1.2072614431381226)
 init edge (2*,3) score=np.float64(3.7362453937530518)
 init edge (3,12*) score=np.float64(2.6624021530151367)
 init edge (5*,12) score=np.float64(2.4297726154327393)
 init edge (6,9*) score=np.float64(2.3572030067443848)
 init edge (48*,39) score=np.float64(2.1846060752868652)
 init edge (48,41*) score=np.float64(2.119276285171509)
 init edge (57*,48) score=np.float64(1.9928776025772095)
 init edge (48,43*) score=np.float64(1.9326719045639038)
 init edge (55*,48) score=np.float64(1.8635882139205933)
 init edge (50*,43) score=np.float64(1.8474559783935547)
 init edge (5,10*) score=np.float64(1.6621935367584229)
 init edge (52*,43) score=np.float64(1.6445786952972412)
 init edge (43,34*) score=np.float64(1.6324530839920044)
 init edge (2,1*) score=np.float64(1.4577800035476685)
 init edge (43,38*) score=np.float64(1.9570790529251099)
 init edge (45*,38) score=np.float64(2.117758274078369)
 init edge (47*,38) score=np.float64(2.043761730194092)
 init edge (47,40*) score=np.float64(2.0032684803009033)
 init edge (45,36*) score=np.float64(1.7433124780654907)
 init edge (33*,36) score=np.float64(1.5633326768875122)
 init edge (36,31*) score=np.float64(1.5161259174346924)
 init edge (36,29*) score=np.float64(1.4566580057144165)
 init edge (36,27*) score=np.float64(1.440176248550415)
 init edge (49*,40) score=np.float64(2.2785987854003906)
 init edge (49,42*) score=np.float64(2.122375011444092)
 init edge (58*,49) score=np.float64(1.9328211545944214)
 init edge (56*,49) score=np.float64(1.8427772521972656)
 init edge (59*,56) score=np.float64(1.90359365940094)
 init edge (59,54*) score=np.float64(1.8612852096557617)
 init edge (59,60*) score=np.float64(1.831799030303955)
Traceback (most recent call last):
  File "/mnt/sdb/wangxinran/monst3r/demo.py", line 424, in <module>
    scene, outfile, imgs = recon_fun(
                           ^^^^^^^^^^
  File "/mnt/sdb/wangxinran/monst3r/demo.py", line 146, in get_reconstructed_scene
    scene.compute_global_alignment(init='mst', niter=niter, schedule=schedule, lr=lr)
  File "/mnt/sdb/wangxinran/anaconda3/envs/monst3r/lib/python3.11/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/sdb/wangxinran/monst3r/dust3r/cloud_opt/base_opt.py", line 404, in compute_global_alignment
    init_fun.init_minimum_spanning_tree(self, save_score_path=save_score_path, save_score_only=save_score_only, niter_PnP=niter_PnP)
  File "/mnt/sdb/wangxinran/anaconda3/envs/monst3r/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/sdb/wangxinran/monst3r/dust3r/cloud_opt/init_im_poses.py", line 98, in init_minimum_spanning_tree
    pts3d, _, im_focals, im_poses = minimum_spanning_tree(self.imshapes, self.edges,
                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/sdb/wangxinran/monst3r/dust3r/cloud_opt/init_im_poses.py", line 294, in minimum_spanning_tree
    res = fast_pnp(pts3d[i], im_focals[i], msk=msk, device=device, niter_PnP=niter_PnP)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/sdb/wangxinran/monst3r/dust3r/cloud_opt/init_im_poses.py", line 420, in fast_pnp
    success, R, T, inliers = cv2.solvePnPRansac(pts3d[msk], pixels[msk], K, None,
                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
cv2.error: OpenCV(4.11.0) /io/opencv/modules/calib3d/src/sqpnp.cpp:253: error: (-215:Assertion failed) point_coordinate_variance >= POINT_VARIANCE_THRESHOLD in function 'computeOmega'

[Junyi42]

Hi @xin-ran-w,

Thanks for raising this issue.

This usually happens when the camera motion is challenging to predict, so that PnP solver has difficulty predicting the initial camera poses. I would recommend trying with the pairwise inference first, and then adjusting the pair graph with the better pairwise results.

Thanks

[xin-ran-w]

Thank you for your reply, what is the pairwise inference. Is there a code example, thank you!