Skip to content

Commit 2c6a946

Browse files
ero1311ero1311
committed
refactored to avoid code duplication in attach image urls
1 parent 912870e commit 2c6a946

File tree

4 files changed

+63
-61
lines changed

4 files changed

+63
-61
lines changed

‎superannotate/common.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@
2929
"Completed": 5,
3030
"Skipped": 6
3131
}
32+
33+
_UPLOAD_STATES = {"Initial": 1, "Basic": 2, "External": 3}
34+
3235
_USER_ROLES = {"Admin": 2, "Annotator": 3, "QA": 4, "Customer": 5, "Viewer": 6}
3336
_AVAILABLE_SEGMENTATION_MODELS = ['autonomous', 'generic']
3437
_MODEL_TRAINING_STATUSES = {
@@ -118,6 +121,10 @@ def annotation_status_str_to_int(annotation_status):
118121
return _ANNOTATION_STATUSES[annotation_status]
119122

120123

124+
def upload_state_str_to_int(upload_state):
125+
return _UPLOAD_STATES[upload_state]
126+
127+
121128
def annotation_status_int_to_str(annotation_status):
122129
"""Converts metadata annotation_status int value to a string
123130

‎superannotate/db/projects.py‎

Lines changed: 54 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -694,9 +694,14 @@ def __upload_images_to_aws_thread(
694694
if len(uploaded_imgs) >= 100:
695695
try:
696696
__create_image(
697-
uploaded_imgs_info[0], uploaded_imgs_info[1], project,
698-
annotation_status, prefix, uploaded_imgs_info[2],
699-
project_folder_id
697+
uploaded_imgs_info[0],
698+
uploaded_imgs_info[1],
699+
project,
700+
annotation_status,
701+
prefix,
702+
uploaded_imgs_info[2],
703+
project_folder_id,
704+
upload_state="Basic"
700705
)
701706
except SABaseException as e:
702707
couldnt_upload[thread_id] += uploaded_imgs
@@ -707,8 +712,14 @@ def __upload_images_to_aws_thread(
707712
uploaded_imgs_info = ([], [], [])
708713
try:
709714
__create_image(
710-
uploaded_imgs_info[0], uploaded_imgs_info[1], project,
711-
annotation_status, prefix, uploaded_imgs_info[2], project_folder_id
715+
uploaded_imgs_info[0],
716+
uploaded_imgs_info[1],
717+
project,
718+
annotation_status,
719+
prefix,
720+
uploaded_imgs_info[2],
721+
project_folder_id,
722+
upload_state="Basic"
712723
)
713724
except SABaseException as e:
714725
couldnt_upload[thread_id] += uploaded_imgs
@@ -718,12 +729,19 @@ def __upload_images_to_aws_thread(
718729

719730

720731
def __create_image(
721-
img_names, img_paths, project, annotation_status, remote_dir, sizes,
722-
project_folder_id
732+
img_names,
733+
img_paths,
734+
project,
735+
annotation_status,
736+
remote_dir,
737+
sizes,
738+
project_folder_id,
739+
upload_state="Initial"
723740
):
724741
if len(img_paths) == 0:
725742
return
726743
team_id, project_id = project["team_id"], project["id"]
744+
upload_state_code = common.upload_state_str_to_int(upload_state)
727745
data = {
728746
"project_id": str(project_id),
729747
"team_id": str(team_id),
@@ -732,14 +750,19 @@ def __create_image(
732750
"team_id": str(team_id),
733751
"images": [],
734752
"annotation_status": annotation_status,
735-
"meta": {}
753+
"meta": {},
754+
"upload_state": upload_state_code
736755
}
737756
if project_folder_id is not None:
738757
data["folder_id"] = project_folder_id
739-
for img_name, img_path, size in zip(img_names, img_paths, sizes):
758+
for img_data, img_path, size in zip(img_names, img_paths, sizes):
740759
img_name_uuid = Path(img_path).name
741760
remote_path = remote_dir + f"{img_name_uuid}"
742-
data["images"].append({"name": img_name, "path": remote_path})
761+
if upload_state == "External":
762+
img_name, img_url = img_data
763+
else:
764+
img_name, img_url = img_data, remote_path
765+
data["images"].append({"name": img_name, "path": img_url})
743766
data["meta"][img_name] = {
744767
"width": size[0],
745768
"height": size[1],
@@ -982,7 +1005,6 @@ def attach_image_urls_to_project(
9821005
daemon=True
9831006
)
9841007
tqdm_thread.start()
985-
9861008
threads = []
9871009
for thread_id in range(_NUM_THREADS):
9881010
t = threading.Thread(
@@ -1030,7 +1052,7 @@ def __attach_image_urls_to_project_thread(
10301052
bucket = s3_resource.Bucket(res["bucket"])
10311053
prefix = res['filePath']
10321054
uploaded_imgs = []
1033-
uploaded_imgs_info = ([], [])
1055+
uploaded_imgs_info = ([], [], [])
10341056
for i in range(start_index, end_index):
10351057
if i >= len_img_paths:
10361058
break
@@ -1051,23 +1073,36 @@ def __attach_image_urls_to_project_thread(
10511073
uploaded_imgs.append(name)
10521074
uploaded_imgs_info[0].append(img_names_urls[i])
10531075
uploaded_imgs_info[1].append(key)
1076+
uploaded_imgs_info[2].append((None, None))
10541077
if len(uploaded_imgs) >= 100:
10551078
try:
1056-
__create_image_url(
1057-
uploaded_imgs_info[0], uploaded_imgs_info[1], project,
1058-
annotation_status, project_folder_id
1079+
__create_image(
1080+
uploaded_imgs_info[0],
1081+
uploaded_imgs_info[1],
1082+
project,
1083+
annotation_status,
1084+
prefix,
1085+
uploaded_imgs_info[2],
1086+
project_folder_id,
1087+
upload_state="External"
10591088
)
10601089
except SABaseException as e:
10611090
couldnt_upload[thread_id] += uploaded_imgs
10621091
logger.warning(e)
10631092
else:
10641093
uploaded[thread_id] += uploaded_imgs
10651094
uploaded_imgs = []
1066-
uploaded_imgs_info = ([], [])
1095+
uploaded_imgs_info = ([], [], [])
10671096
try:
1068-
__create_image_url(
1069-
uploaded_imgs_info[0], uploaded_imgs_info[1], project,
1070-
annotation_status, project_folder_id
1097+
__create_image(
1098+
uploaded_imgs_info[0],
1099+
uploaded_imgs_info[1],
1100+
project,
1101+
annotation_status,
1102+
prefix,
1103+
uploaded_imgs_info[2],
1104+
project_folder_id,
1105+
upload_state="External"
10711106
)
10721107
except SABaseException as e:
10731108
couldnt_upload[thread_id] += uploaded_imgs
@@ -1076,45 +1111,6 @@ def __attach_image_urls_to_project_thread(
10761111
uploaded[thread_id] += uploaded_imgs
10771112

10781113

1079-
def __create_image_url(
1080-
img_names_urls, remote_paths, project, annotation_status, project_folder_id
1081-
):
1082-
if len(remote_paths) == 0:
1083-
return
1084-
team_id, project_id = project["team_id"], project["id"]
1085-
data = {
1086-
"project_id": str(project_id),
1087-
"team_id": str(team_id),
1088-
"images": [],
1089-
"annotation_status": annotation_status,
1090-
"meta": {},
1091-
"upload_state": 2
1092-
}
1093-
if project_folder_id is not None:
1094-
data["folder_id"] = project_folder_id
1095-
for img_name_url, remote_path in zip(img_names_urls, remote_paths):
1096-
data["images"].append(
1097-
{
1098-
"name": img_name_url[0],
1099-
"path": img_name_url[1]
1100-
}
1101-
)
1102-
data["meta"][img_name_url[0]] = {
1103-
"width": None,
1104-
"height": None,
1105-
"annotation_json_path": remote_path + ".json",
1106-
"annotation_bluemap_path": remote_path + ".png"
1107-
}
1108-
1109-
response = _api.send_request(
1110-
req_type='POST', path='/image/ext-create', json_req=data
1111-
)
1112-
if not response.ok:
1113-
raise SABaseException(
1114-
response.status_code, "Couldn't ext-create image " + response.text
1115-
)
1116-
1117-
11181114
def upload_images_from_public_urls_to_project(
11191115
project,
11201116
img_urls,

‎tests/attach_urls.csv‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,4 @@ https://drive.google.com/uc?export=download&id=1SfGcn9hdkVM35ZP0S93eStsE7Ti4GtHU
66
https://drive.google.com/uc?export=download&id=1SfGcn9hdkVM35ZP0S93eStsE7Ti4GtHU,6022a74b5384c50017c366cv
77
https://drive.google.com/uc?export=download&id=1geS2YtQiTYuiduEirKVYxBujHJaIWA3V,
88
https://drive.google.com/uc?export=download&id=1geS2YtQiTYuiduEirKVYxBujHJaIWA3V,6022a74b26aec4002575b9rm
9-
https://drive.google.com/uc?export=download&id=1geS2YtQiTYuiduEirKVYxBujHJaIWA3V,6022a74b26aec4002575b9e8
109

‎tests/test_attach_image_urls.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def test_attach_image_urls():
1919
project, PATH_TO_URLS
2020
)
2121

22-
assert len(uploaded) == 8
22+
assert len(uploaded) == 7
2323
assert len(could_not_upload) == 0
2424
assert len(existing_images) == 0
2525

@@ -29,4 +29,4 @@ def test_attach_image_urls():
2929

3030
assert len(uploaded) == 2
3131
assert len(could_not_upload) == 0
32-
assert len(existing_images) == 6
32+
assert len(existing_images) == 5

0 commit comments

Comments
 (0)