From 35ecdae2679b63ba85a8b596d4223d4450ba42f1 Mon Sep 17 00:00:00 2001 From: KJStick Date: Wed, 15 Mar 2023 09:39:46 -0400 Subject: [PATCH 01/30] added request ip initial feature --- bot.py | 4 +++ features/awx.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ 3 files changed, 70 insertions(+) diff --git a/bot.py b/bot.py index d23a104a..612adce1 100644 --- a/bot.py +++ b/bot.py @@ -48,6 +48,7 @@ "**Reset AWS key** > delete key and create new key\n", "**Delete AWS key** > deletes access key\n", "**AWS key status** > shows the status of all aws access keys\n", + "**Request ip address** > allocates an static ip address\n", "**Create accounts** > create base COLAB accounts\n", "**Delete accounts** > delete COLAB accounts\n", "**Reset passwords** > resets all COLAB associated passwords\n", @@ -306,6 +307,9 @@ async def process(self, req: Request): elif self.activity.get("text") == "delete accounts": await awx.delete_accounts(self.activity) + elif self.activity.get("text") == "request ip addresss": + await awx.request_ip(self.activity) + elif ( self.activity.get("text")[:3] == "cml" ): # Add searches for cml dialogue here diff --git a/features/awx.py b/features/awx.py index 8cb8ddf8..4f100b44 100644 --- a/features/awx.py +++ b/features/awx.py @@ -5,12 +5,14 @@ import re import tempfile from datetime import datetime, date +import ipaddress from cryptography.fernet import Fernet import aiohttp import pymongo import urllib3 import boto3 from boto3.dynamodb.conditions import Key +import pynetbox import yaml from virl2_client import ClientLibrary from jinja2 import Template @@ -1083,3 +1085,65 @@ async def get_iam_user(iam_username, iam=None): return return user + + +async def request_ip(activity): + """Allocates a static ip from netbox for lab use""" + date_format = "%m/%d/%Y" + url = "https://netbox3.aws.ciscops.net" + token = "0123456789abcdef0123456789abcdef01234567" + + username = activity["sender_email"].split("@")[0] + + nb = pynetbox.api(url, token) + + # Find static ip pool + ip_ranges = nb.ipam.ip_ranges.all() + ip_range = None + for ip_range in ip_ranges: + if "Static IPs" in ip_range["description"]: + break + + # ip_type = ip_range.family.label + # check if ipv4 or 6 - below assumes 4 + + start_address = ip_range.start_address + mask = start_address[-3:] + net = ipaddress.ip_network(start_address, False) + + # Find first available ip + valid_ip = False + for ip in net: + ip = str(ip) + mask + + # make sure to only check at start of range, not start of net + if ip == start_address: + valid_ip = True + if not valid_ip: + continue + + address = nb.ipam.ip_addresses.get(address=ip) + if address is None: + # Not made - so create + address = nb.ipam.ip_addresses.create(get_ipv4_dict(ip)) + break + + if address.custom_fields["username_assigned"] is None: + break + + if address is None: + raise Exception("There are no more ips available") + + # assign to user + address.custom_fields["username_assigned"] = username + address.custom_fields["date_last_used"] = date.today().strftime(date_format) + address.save() + + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + message = f"""New static IP Address assigned: { address }""" + await webex.post_message_to_webex(message) + + +def get_ipv4_dict(ip_address: str): + """Helper function for static ip requests""" + return {"family": 4, "address": ip_address, "vrf": None} diff --git a/requirements.txt b/requirements.txt index 9460abd1..5a92ac4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ webexteamssdk==1.6 virl2_client==2.4.0 PyYAML==5.4 cryptography==39.0.1 +ipaddress==1.0.23 +pynetbox==7.0.1 From 3e2a2a9b8a70cb5bfd45e925b3ad7c448bd99f03 Mon Sep 17 00:00:00 2001 From: KJStick Date: Wed, 15 Mar 2023 09:46:19 -0400 Subject: [PATCH 02/30] redid exception to webex msg --- features/awx.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/features/awx.py b/features/awx.py index 4f100b44..4342179d 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1089,6 +1089,7 @@ async def get_iam_user(iam_username, iam=None): async def request_ip(activity): """Allocates a static ip from netbox for lab use""" + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) date_format = "%m/%d/%Y" url = "https://netbox3.aws.ciscops.net" token = "0123456789abcdef0123456789abcdef01234567" @@ -1132,14 +1133,14 @@ async def request_ip(activity): break if address is None: - raise Exception("There are no more ips available") + await webex.post_message_to_webex("There are no more ips available") + return # assign to user address.custom_fields["username_assigned"] = username address.custom_fields["date_last_used"] = date.today().strftime(date_format) address.save() - webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) message = f"""New static IP Address assigned: { address }""" await webex.post_message_to_webex(message) From c41d405c973d19645cea3588878a2afd90856ed5 Mon Sep 17 00:00:00 2001 From: KJStick Date: Sat, 18 Mar 2023 18:03:07 -0400 Subject: [PATCH 03/30] added dynamo --- bot.py | 2 +- features/awx.py | 64 +++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/bot.py b/bot.py index 612adce1..616e4040 100644 --- a/bot.py +++ b/bot.py @@ -41,6 +41,7 @@ "**CML show IP addresses** > show IP addresses\n", "**CML show server utilization** > show current CPU and Memory usage\n", "**CML stop lab** > stop labs of your choice\n", + "**Request static IP address** > allocates an static ip address for CML\n", "**Create AWS account** > create AWS COLAB account\n", "**Create VPN account** > create an AnyConnect to COLAB VPN account\n", "**Create AWS key** > create aws access key\n", @@ -48,7 +49,6 @@ "**Reset AWS key** > delete key and create new key\n", "**Delete AWS key** > deletes access key\n", "**AWS key status** > shows the status of all aws access keys\n", - "**Request ip address** > allocates an static ip address\n", "**Create accounts** > create base COLAB accounts\n", "**Delete accounts** > delete COLAB accounts\n", "**Reset passwords** > resets all COLAB associated passwords\n", diff --git a/features/awx.py b/features/awx.py index 4342179d..138e252f 100644 --- a/features/awx.py +++ b/features/awx.py @@ -11,7 +11,7 @@ import pymongo import urllib3 import boto3 -from boto3.dynamodb.conditions import Key +from boto3.dynamodb.conditions import Key, Attr import pynetbox import yaml from virl2_client import ClientLibrary @@ -1089,22 +1089,38 @@ async def get_iam_user(iam_username, iam=None): async def request_ip(activity): """Allocates a static ip from netbox for lab use""" - webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) date_format = "%m/%d/%Y" + date_string = date.today().strftime(date_format) url = "https://netbox3.aws.ciscops.net" token = "0123456789abcdef0123456789abcdef01234567" - username = activity["sender_email"].split("@")[0] + ## APIs nb = pynetbox.api(url, token) - # Find static ip pool + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + + dynamodb = boto3.resource( + "dynamodb", + region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod + aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, + aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, + ) + + table = dynamodb.Table( + "colab_directory" # Table Name + ) # TODO remove dev extension when pushing to prod + + # Find static ip pool on netbox ip_ranges = nb.ipam.ip_ranges.all() ip_range = None for ip_range in ip_ranges: if "Static IPs" in ip_range["description"]: break - + if ip_range is None: + message = f"""No IP pool could be found on Netbox""" + await webex.post_message_to_webex(message) + return False # ip_type = ip_range.family.label # check if ipv4 or 6 - below assumes 4 @@ -1125,7 +1141,7 @@ async def request_ip(activity): address = nb.ipam.ip_addresses.get(address=ip) if address is None: - # Not made - so create + # IP not made on netbox - so create address = nb.ipam.ip_addresses.create(get_ipv4_dict(ip)) break @@ -1134,17 +1150,45 @@ async def request_ip(activity): if address is None: await webex.post_message_to_webex("There are no more ips available") - return + return False - # assign to user + # assign to user on netbox address.custom_fields["username_assigned"] = username - address.custom_fields["date_last_used"] = date.today().strftime(date_format) + address.custom_fields["date_last_used"] = date_string address.save() + ## insert ip into database + + # check to see if ip field already there + response = table.query( + KeyConditionExpression=Key("email").eq(activity["sender_email"]), + FilterExpression=Attr("ip_addresses").exists(), + ) + + # create ip field map if it doesn't exist + if response["Count"] == 0: + table.update_item( + Key={"email": activity["sender_email"]}, + UpdateExpression="SET #ip_addresses= :value", + ExpressionAttributeNames={"#ip_addresses": "ip_addresses"}, + ExpressionAttributeValues={":value": ""}, + ) + + # insert new ip + table.update_item( + Key={"email": activity["sender_email"]}, + UpdateExpression="SET #ip_addresses.#ip_address= :ip_data", + ExpressionAttributeNames={ + "#ip_addresses": "ip_addresses", + "#ip_address": address, + }, + ExpressionAttributeValues={":ip_data": date_string}, + ) + + # message user message = f"""New static IP Address assigned: { address }""" await webex.post_message_to_webex(message) - def get_ipv4_dict(ip_address: str): """Helper function for static ip requests""" return {"family": 4, "address": ip_address, "vrf": None} From 63b64129c8550c24e7e421d49c0d79f8d54c6c99 Mon Sep 17 00:00:00 2001 From: KJStick Date: Tue, 21 Mar 2023 16:01:25 -0400 Subject: [PATCH 04/30] redid variales for config --- bot.py | 2 +- features/awx.py | 54 +++++++++++++++++++++++-------------------------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/bot.py b/bot.py index 616e4040..be08c41b 100644 --- a/bot.py +++ b/bot.py @@ -307,7 +307,7 @@ async def process(self, req: Request): elif self.activity.get("text") == "delete accounts": await awx.delete_accounts(self.activity) - elif self.activity.get("text") == "request ip addresss": + elif self.activity.get("text") == "request ip": await awx.request_ip(self.activity) elif ( diff --git a/features/awx.py b/features/awx.py index 138e252f..f22587d6 100644 --- a/features/awx.py +++ b/features/awx.py @@ -642,16 +642,7 @@ async def handle_labbing_card(activity): labs_to_save = [] labs_to_delete = [] - dynamodb = boto3.resource( - "dynamodb", - region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod - aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, - aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, - ) - - table = dynamodb.Table( - "colab_directory" # Table Name - ) # TODO remove dev extension when pushing to prod + table = get_dynamo_colab_table() cml_server = CONFIG.SERVER_LIST.split(",")[0] user_and_domain = user_email.split("@") @@ -1088,40 +1079,29 @@ async def get_iam_user(iam_username, iam=None): async def request_ip(activity): - """Allocates a static ip from netbox for lab use""" - date_format = "%m/%d/%Y" - date_string = date.today().strftime(date_format) - url = "https://netbox3.aws.ciscops.net" - token = "0123456789abcdef0123456789abcdef01234567" + """Allocates a static ip from netbox for cml lab use""" + date_string = str(int(datetime.timestamp(datetime.now()))) + url = CONFIG.NETBOX_URL + token = CONFIG.NETBOX_TOKEN username = activity["sender_email"].split("@")[0] ## APIs nb = pynetbox.api(url, token) - webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) - - dynamodb = boto3.resource( - "dynamodb", - region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod - aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, - aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, - ) - - table = dynamodb.Table( - "colab_directory" # Table Name - ) # TODO remove dev extension when pushing to prod + table = get_dynamo_colab_table() # Find static ip pool on netbox ip_ranges = nb.ipam.ip_ranges.all() ip_range = None for ip_range in ip_ranges: - if "Static IPs" in ip_range["description"]: + if "static ips" in ip_range["description"].lower(): break if ip_range is None: message = f"""No IP pool could be found on Netbox""" await webex.post_message_to_webex(message) return False - # ip_type = ip_range.family.label + + #ip_type = ip_range.family.label # check if ipv4 or 6 - below assumes 4 start_address = ip_range.start_address @@ -1145,6 +1125,7 @@ async def request_ip(activity): address = nb.ipam.ip_addresses.create(get_ipv4_dict(ip)) break + # ip created but not assigned if address.custom_fields["username_assigned"] is None: break @@ -1192,3 +1173,18 @@ async def request_ip(activity): def get_ipv4_dict(ip_address: str): """Helper function for static ip requests""" return {"family": 4, "address": ip_address, "vrf": None} + +def get_dynamo_colab_table(): + """Returns dynamo colab table""" + dynamodb = boto3.resource( + "dynamodb", + region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod + aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, + aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, + ) + + table = dynamodb.Table( + CONFIG.AWS_DYNAMO_TABLE # Table Name + ) + + return table From 62f72b11ef512378cd5c340f57b7d2124737b79e Mon Sep 17 00:00:00 2001 From: KJStick Date: Wed, 22 Mar 2023 22:58:20 -0400 Subject: [PATCH 05/30] cast config gets to str --- features/awx.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/features/awx.py b/features/awx.py index f22587d6..0d0f184e 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1081,8 +1081,9 @@ async def get_iam_user(iam_username, iam=None): async def request_ip(activity): """Allocates a static ip from netbox for cml lab use""" date_string = str(int(datetime.timestamp(datetime.now()))) - url = CONFIG.NETBOX_URL - token = CONFIG.NETBOX_TOKEN + url = str(CONFIG.NETBOX_URL) + logging.info("MY AWESOME URL: %s", url) + token = str(CONFIG.NETBOX_TOKEN) username = activity["sender_email"].split("@")[0] ## APIs From ae5e328e7a7a5e2c6041a10e35361e5cd183b53a Mon Sep 17 00:00:00 2001 From: KJStick Date: Wed, 22 Mar 2023 23:44:25 -0400 Subject: [PATCH 06/30] updated configs for consistency --- .github/workflows/ci_dev.yaml | 10 +++++----- .github/workflows/ci_prod.yaml | 6 +++--- colabot-manifest-dev.yaml.j2 | 8 ++++---- colabot-manifest-prod.yaml.j2 | 6 +++--- colabot-secrets-dev.yaml.j2 | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci_dev.yaml b/.github/workflows/ci_dev.yaml index ac33d650..79ce0c85 100644 --- a/.github/workflows/ci_dev.yaml +++ b/.github/workflows/ci_dev.yaml @@ -86,7 +86,7 @@ jobs: MONGO_DB_ROOT_PASSWORD: ${{ secrets.MONGO_DB_ROOT_PASSWORD }} AWX_PASSWORD: ${{ secrets.AWX_PASSWORD }} NLP_SECRET: ${{ secrets.NLP_SECRET }} - AWS_PASSWORD: ${{ secrets.AWS_PASSWORD }} + AWS_PASSWORD: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} BOT_ID_DEV: ${{ secrets.BOT_ID_DEV }} BOT_NAME_DEV: ${{ secrets.BOT_NAME_DEV }} AUTHORIZED_ROOMS_DEV: ${{ secrets.AUTHORIZED_ROOMS_DEV }} @@ -100,11 +100,11 @@ jobs: NLP_SERVER_DEV: ${{ secrets.NLP_SERVER_DEV }} VCENTER_SERVER: ${{ secrets.VCENTER_SERVER }} ADMINISTRATORS_DEV: ${{ secrets.ADMINISTRATORS_DEV }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_REGION_COLAB: ${{ secrets.AWS_REGION_COLAB }} - AWS_ACCESS_KEY_ID_COLAB: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} - AWS_PASSWORD_COLAB: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} COLABOT_SECRET: ${{ secrets.COLABOT_SECRET }} + DYNAMO_TABLE: ${{ secrets.DYNAMO_TABLE_DEV }} + NETBOX_URL: ${{ secrets.NETBOX_URL_DEV }} + NETBOX_TOKEN: ${{ secrets.NETBOX_TOKEN_DEV }} run: python3 process-j2.py - name: Apply and rollout diff --git a/.github/workflows/ci_prod.yaml b/.github/workflows/ci_prod.yaml index f5cd6c75..00095f56 100644 --- a/.github/workflows/ci_prod.yaml +++ b/.github/workflows/ci_prod.yaml @@ -101,10 +101,10 @@ jobs: VCENTER_SERVER: ${{ secrets.VCENTER_SERVER }} ADMINISTRATORS_PROD: ${{ secrets.ADMINISTRATORS_PROD }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_REGION_COLAB: ${{ secrets.AWS_REGION_COLAB }} - AWS_ACCESS_KEY_ID_COLAB: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} - AWS_PASSWORD_COLAB: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} COLABOT_SECRET: ${{ secrets.COLABOT_SECRET }} + DYNAMO_TABLE: ${{ secrets.DYNAMO_TABLE_PROD }} + NETBOX_URL: ${{ secrets.NETBOX_URL_PROD }} + NETBOX_TOKEN: ${{ secrets.NETBOX_TOKEN_PROD }} run: python3 process-j2.py - name: Apply and rollout diff --git a/colabot-manifest-dev.yaml.j2 b/colabot-manifest-dev.yaml.j2 index dd8487f9..62480514 100644 --- a/colabot-manifest-dev.yaml.j2 +++ b/colabot-manifest-dev.yaml.j2 @@ -110,7 +110,7 @@ spec: - name: ADMINISTRATORS value: {{ ADMINISTRATORS_DEV }} - name: AWS_ACCESS_KEY_ID - value: {{ AWS_ACCESS_KEY_ID_COLAB }} + value: {{ AWS_ACCESS_KEY_ID }} - name: AWS_REGION value: 'us-east-1' - name: AWS_SECRET_ACCESS_KEY @@ -121,11 +121,11 @@ spec: - name: COLABOT_SECRET value: {{ COLABOT_SECRET }} - name: AWS_DYNAMO_TABLE - value: {{ DYNAMO_TABLE_DEV }} + value: {{ DYNAMO_TABLE }} - name: NETBOX_URL - value: {{ NETBOX_URL_DEV }} + value: {{ NETBOX_URL }} - name: NETBOX_TOKEN - value: {{ NETBOX_TOKEN_DEV }} + value: {{ NETBOX_TOKEN }} --- diff --git a/colabot-manifest-prod.yaml.j2 b/colabot-manifest-prod.yaml.j2 index d3a5d5f4..79b49cba 100644 --- a/colabot-manifest-prod.yaml.j2 +++ b/colabot-manifest-prod.yaml.j2 @@ -114,11 +114,11 @@ spec: - name: COLABOT_SECRET value: {{ COLABOT_SECRET }} - name: AWS_DYNAMO_TABLE - value: {{ DYNAMO_TABLE_PROD }} + value: {{ DYNAMO_TABLE }} - name: NETBOX_URL - value: {{ NETBOX_URL_PROD }} + value: {{ NETBOX_URL }} - name: NETBOX_TOKEN - value: {{ NETBOX_TOKEN_PROD }} + value: {{ NETBOX_TOKEN }} --- diff --git a/colabot-secrets-dev.yaml.j2 b/colabot-secrets-dev.yaml.j2 index d4cd6226..3e5f31de 100644 --- a/colabot-secrets-dev.yaml.j2 +++ b/colabot-secrets-dev.yaml.j2 @@ -13,4 +13,4 @@ stringData: mongo_initb_root_password: {{ MONGO_DB_ROOT_PASSWORD }} awx_password: {{ AWX_PASSWORD }} nlp_secret: {{ NLP_SECRET }} - aws_password: {{ AWS_PASSWORD_COLAB }} + aws_password: {{ AWS_PASSWORD }} From 24eeaec39f3a3f14ce24e735744b3b2a7a883495 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 23 Mar 2023 00:02:39 -0400 Subject: [PATCH 07/30] updated dynamo to mapping --- features/awx.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/features/awx.py b/features/awx.py index 0d0f184e..d1149d3d 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1098,7 +1098,7 @@ async def request_ip(activity): if "static ips" in ip_range["description"].lower(): break if ip_range is None: - message = f"""No IP pool could be found on Netbox""" + message = """No IP pool could be found on Netbox""" await webex.post_message_to_webex(message) return False @@ -1153,7 +1153,7 @@ async def request_ip(activity): Key={"email": activity["sender_email"]}, UpdateExpression="SET #ip_addresses= :value", ExpressionAttributeNames={"#ip_addresses": "ip_addresses"}, - ExpressionAttributeValues={":value": ""}, + ExpressionAttributeValues={":value": {}}, ) # insert new ip @@ -1164,7 +1164,11 @@ async def request_ip(activity): "#ip_addresses": "ip_addresses", "#ip_address": address, }, - ExpressionAttributeValues={":ip_data": date_string}, + ExpressionAttributeValues={ + ":ip_data": { + "date_last_used": date_string, + } + }, ) # message user From 7d8b494fd80b08e95703e5d3ffa0f67e70073d8c Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 23 Mar 2023 00:28:48 -0400 Subject: [PATCH 08/30] undo table fx call --- features/awx.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index d1149d3d..a231192c 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1089,7 +1089,17 @@ async def request_ip(activity): ## APIs nb = pynetbox.api(url, token) webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) - table = get_dynamo_colab_table() + #table = get_dynamo_colab_table() + dynamodb = boto3.resource( + "dynamodb", + region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod + aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, + aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, + ) + + table = dynamodb.Table( + CONFIG.AWS_DYNAMO_TABLE # Table Name + ) # Find static ip pool on netbox ip_ranges = nb.ipam.ip_ranges.all() From 1b01b1c1f2f392824e4d779d4aeafe7a4af379ec Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 23 Mar 2023 00:49:05 -0400 Subject: [PATCH 09/30] made address to str --- features/awx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index a231192c..02264d14 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1172,7 +1172,7 @@ async def request_ip(activity): UpdateExpression="SET #ip_addresses.#ip_address= :ip_data", ExpressionAttributeNames={ "#ip_addresses": "ip_addresses", - "#ip_address": address, + "#ip_address": str(address), }, ExpressionAttributeValues={ ":ip_data": { From b54f42ad9668fb6b08964db59b6cdbcbefb71efc Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 23 Mar 2023 00:57:45 -0400 Subject: [PATCH 10/30] changed webex --- config.py | 1 - features/awx.py | 40 ++++++++++++++++++++-------------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/config.py b/config.py index 27ceda12..d99ca103 100644 --- a/config.py +++ b/config.py @@ -48,4 +48,3 @@ class DefaultConfig: COLABOT_CYPHER = os.environ.get("COLABOT_SECRET") NETBOX_URL = os.environ.get("NETBOX_URL") NETBOX_TOKEN = os.environ.get("NETBOX_TOKEN") - diff --git a/features/awx.py b/features/awx.py index 02264d14..93797b65 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1089,17 +1089,7 @@ async def request_ip(activity): ## APIs nb = pynetbox.api(url, token) webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) - #table = get_dynamo_colab_table() - dynamodb = boto3.resource( - "dynamodb", - region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod - aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, - aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, - ) - - table = dynamodb.Table( - CONFIG.AWS_DYNAMO_TABLE # Table Name - ) + table = get_dynamo_colab_table() # Find static ip pool on netbox ip_ranges = nb.ipam.ip_ranges.all() @@ -1108,11 +1098,14 @@ async def request_ip(activity): if "static ips" in ip_range["description"].lower(): break if ip_range is None: - message = """No IP pool could be found on Netbox""" + message = dict( + text="No IP pool could be found on Netbox", + toPersonId=activity["sender"], + ) await webex.post_message_to_webex(message) return False - - #ip_type = ip_range.family.label + + # ip_type = ip_range.family.label # check if ipv4 or 6 - below assumes 4 start_address = ip_range.start_address @@ -1141,7 +1134,11 @@ async def request_ip(activity): break if address is None: - await webex.post_message_to_webex("There are no more ips available") + message = dict( + text="There are no more ips available", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) return False # assign to user on netbox @@ -1150,7 +1147,7 @@ async def request_ip(activity): address.save() ## insert ip into database - + # check to see if ip field already there response = table.query( KeyConditionExpression=Key("email").eq(activity["sender_email"]), @@ -1182,13 +1179,18 @@ async def request_ip(activity): ) # message user - message = f"""New static IP Address assigned: { address }""" + message = dict( + text=f"New static IP Address assigned: { str(address) }", + toPersonId=activity["sender"], + ) await webex.post_message_to_webex(message) + def get_ipv4_dict(ip_address: str): """Helper function for static ip requests""" return {"family": 4, "address": ip_address, "vrf": None} + def get_dynamo_colab_table(): """Returns dynamo colab table""" dynamodb = boto3.resource( @@ -1198,8 +1200,6 @@ def get_dynamo_colab_table(): aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, ) - table = dynamodb.Table( - CONFIG.AWS_DYNAMO_TABLE # Table Name - ) + table = dynamodb.Table(CONFIG.AWS_DYNAMO_TABLE) # Table Name return table From 354a3db9fc5d87bd8e86634ded6ee91d47adae3f Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 23 Mar 2023 22:13:22 -0400 Subject: [PATCH 11/30] added safety features --- features/awx.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/features/awx.py b/features/awx.py index 93797b65..fdd40534 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1080,6 +1080,7 @@ async def get_iam_user(iam_username, iam=None): async def request_ip(activity): """Allocates a static ip from netbox for cml lab use""" + ip_limit = 10 date_string = str(int(datetime.timestamp(datetime.now()))) url = str(CONFIG.NETBOX_URL) logging.info("MY AWESOME URL: %s", url) @@ -1091,6 +1092,20 @@ async def request_ip(activity): webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) table = get_dynamo_colab_table() + ## make sure user under ip limit + response = table.query( + KeyConditionExpression=Key("email").eq(activity["sender_email"]) + ) + + if "ip_addresses" in response["Items"][0] and len(response["Items"][0]["ip_addresses"]) >= ip_limit: + message = dict( + text=f"You have reached the limit of { ip_limit } reserved ip addresses", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # Find static ip pool on netbox ip_ranges = nb.ipam.ip_ranges.all() ip_range = None @@ -1130,7 +1145,7 @@ async def request_ip(activity): break # ip created but not assigned - if address.custom_fields["username_assigned"] is None: + if address.description == "" and address.status == "active": break if address is None: @@ -1142,8 +1157,8 @@ async def request_ip(activity): return False # assign to user on netbox - address.custom_fields["username_assigned"] = username - address.custom_fields["date_last_used"] = date_string + address.description = username + address.status = 'reserved' address.save() ## insert ip into database @@ -1185,6 +1200,8 @@ async def request_ip(activity): ) await webex.post_message_to_webex(message) + return True + def get_ipv4_dict(ip_address: str): """Helper function for static ip requests""" From 8c9678b508c46cfe26ec067ee10552206807c974 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 16:32:08 -0400 Subject: [PATCH 12/30] refactored code --- features/awx.py | 98 ++++++++++++++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/features/awx.py b/features/awx.py index fdd40534..a27fb1d1 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1081,23 +1081,19 @@ async def get_iam_user(iam_username, iam=None): async def request_ip(activity): """Allocates a static ip from netbox for cml lab use""" ip_limit = 10 - date_string = str(int(datetime.timestamp(datetime.now()))) - url = str(CONFIG.NETBOX_URL) - logging.info("MY AWESOME URL: %s", url) - token = str(CONFIG.NETBOX_TOKEN) + nb_url = str(CONFIG.NETBOX_URL) + nb_token = str(CONFIG.NETBOX_TOKEN) username = activity["sender_email"].split("@")[0] ## APIs - nb = pynetbox.api(url, token) + nb = pynetbox.api(nb_url, nb_token) webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) table = get_dynamo_colab_table() ## make sure user under ip limit - response = table.query( - KeyConditionExpression=Key("email").eq(activity["sender_email"]) - ) + ip_addresses = get_ips_dynamo(table, activity["sender_email"]) - if "ip_addresses" in response["Items"][0] and len(response["Items"][0]["ip_addresses"]) >= ip_limit: + if len(ip_addresses) >= ip_limit: message = dict( text=f"You have reached the limit of { ip_limit } reserved ip addresses", toPersonId=activity["sender"], @@ -1105,13 +1101,14 @@ async def request_ip(activity): await webex.post_message_to_webex(message) return False - # Find static ip pool on netbox ip_ranges = nb.ipam.ip_ranges.all() ip_range = None for ip_range in ip_ranges: if "static ips" in ip_range["description"].lower(): break + ip_range = None + if ip_range is None: message = dict( text="No IP pool could be found on Netbox", @@ -1123,6 +1120,41 @@ async def request_ip(activity): # ip_type = ip_range.family.label # check if ipv4 or 6 - below assumes 4 + address = get_available_ip(nb, ip_range) + + if address is None: + message = dict( + text="There are no more ips available", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # assign to user on netbox + address.description = username + address.status = "reserved" + address.save() + + ## insert ip into database + update_ip_dynamo(table, activity["sender_email"], address) + + # message user + message = dict( + text=f"New static IP Address assigned: { str(address) }", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + + return True + + +def get_ipv4_dict(ip_address: str): + """Helper function for static ip requests""" + return {"family": 4, "address": ip_address, "vrf": None} + + +def get_available_ip(nb: pynetbox.api, ip_range: pynetbox.ipam.ip_range): + """Returns the next available ip address as a Netbox ip object""" start_address = ip_range.start_address mask = start_address[-3:] net = ipaddress.ip_network(start_address, False) @@ -1144,35 +1176,31 @@ async def request_ip(activity): address = nb.ipam.ip_addresses.create(get_ipv4_dict(ip)) break - # ip created but not assigned + # ip created but not assigned - what we want if address.description == "" and address.status == "active": break - if address is None: - message = dict( - text="There are no more ips available", - toPersonId=activity["sender"], - ) - await webex.post_message_to_webex(message) - return False + address = None - # assign to user on netbox - address.description = username - address.status = 'reserved' - address.save() + return address - ## insert ip into database + +def update_ip_dynamo(table, user_email: str, ip_address: str, date_string: str = None): + """Creates or updates a ip address with the date string in dynamo""" + + if date_string is None: + date_string = str(int(datetime.timestamp(datetime.now()))) # check to see if ip field already there response = table.query( - KeyConditionExpression=Key("email").eq(activity["sender_email"]), + KeyConditionExpression=Key("email").eq(user_email), FilterExpression=Attr("ip_addresses").exists(), ) # create ip field map if it doesn't exist if response["Count"] == 0: table.update_item( - Key={"email": activity["sender_email"]}, + Key={"email": user_email}, UpdateExpression="SET #ip_addresses= :value", ExpressionAttributeNames={"#ip_addresses": "ip_addresses"}, ExpressionAttributeValues={":value": {}}, @@ -1180,11 +1208,11 @@ async def request_ip(activity): # insert new ip table.update_item( - Key={"email": activity["sender_email"]}, + Key={"email": user_email}, UpdateExpression="SET #ip_addresses.#ip_address= :ip_data", ExpressionAttributeNames={ "#ip_addresses": "ip_addresses", - "#ip_address": str(address), + "#ip_address": str(ip_address), }, ExpressionAttributeValues={ ":ip_data": { @@ -1193,19 +1221,15 @@ async def request_ip(activity): }, ) - # message user - message = dict( - text=f"New static IP Address assigned: { str(address) }", - toPersonId=activity["sender"], - ) - await webex.post_message_to_webex(message) - return True +def get_ips_dynamo(table, user_email): + """Returns the ip addresses associated with a user""" + response = table.query(KeyConditionExpression=Key("email").eq(user_email)) + if "ip_addresses" not in response["Items"][0]: + return None -def get_ipv4_dict(ip_address: str): - """Helper function for static ip requests""" - return {"family": 4, "address": ip_address, "vrf": None} + return response["Items"][0]["ip_addresses"] def get_dynamo_colab_table(): From 99e5ba0567aa47ddb955b9980718e7d40df90173 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 16:42:40 -0400 Subject: [PATCH 13/30] added logging and fixed bug --- features/awx.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index a27fb1d1..da8798a0 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1080,6 +1080,8 @@ async def get_iam_user(iam_username, iam=None): async def request_ip(activity): """Allocates a static ip from netbox for cml lab use""" + + logging.info("Start Request IP") ip_limit = 10 nb_url = str(CONFIG.NETBOX_URL) nb_token = str(CONFIG.NETBOX_TOKEN) @@ -1089,11 +1091,13 @@ async def request_ip(activity): nb = pynetbox.api(nb_url, nb_token) webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) table = get_dynamo_colab_table() + logging.warning("%s", str(type(table))) ## make sure user under ip limit ip_addresses = get_ips_dynamo(table, activity["sender_email"]) if len(ip_addresses) >= ip_limit: + logging.info("User %s has reached limit of %s", username, str(ip_limit)) message = dict( text=f"You have reached the limit of { ip_limit } reserved ip addresses", toPersonId=activity["sender"], @@ -1116,6 +1120,7 @@ async def request_ip(activity): ) await webex.post_message_to_webex(message) return False + logging.info("Found IP range %s", str(ip_range)) # ip_type = ip_range.family.label # check if ipv4 or 6 - below assumes 4 @@ -1134,6 +1139,7 @@ async def request_ip(activity): address.description = username address.status = "reserved" address.save() + logging.info("Saved IP on netbox") ## insert ip into database update_ip_dynamo(table, activity["sender_email"], address) @@ -1153,8 +1159,11 @@ def get_ipv4_dict(ip_address: str): return {"family": 4, "address": ip_address, "vrf": None} -def get_available_ip(nb: pynetbox.api, ip_range: pynetbox.ipam.ip_range): +def get_available_ip( + nb: pynetbox.core.api.Api, ip_range: pynetbox.models.ipam.IpRanges +): """Returns the next available ip address as a Netbox ip object""" + logging.info("Finding next available ip") start_address = ip_range.start_address mask = start_address[-3:] net = ipaddress.ip_network(start_address, False) @@ -1221,6 +1230,8 @@ def update_ip_dynamo(table, user_email: str, ip_address: str, date_string: str = }, ) + logging.info("Updated IP on dynamo") + def get_ips_dynamo(table, user_email): """Returns the ip addresses associated with a user""" From 44bbdfd7ffbc6c0560e55c7db76fc7fdd0a78294 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 16:49:39 -0400 Subject: [PATCH 14/30] added logging and docs --- features/awx.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/features/awx.py b/features/awx.py index da8798a0..1ec61ca9 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1091,7 +1091,6 @@ async def request_ip(activity): nb = pynetbox.api(nb_url, nb_token) webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) table = get_dynamo_colab_table() - logging.warning("%s", str(type(table))) ## make sure user under ip limit ip_addresses = get_ips_dynamo(table, activity["sender_email"]) @@ -1126,6 +1125,7 @@ async def request_ip(activity): # check if ipv4 or 6 - below assumes 4 address = get_available_ip(nb, ip_range) + logging.info("Got IP address %s", str(address)) if address is None: message = dict( @@ -1162,6 +1162,7 @@ def get_ipv4_dict(ip_address: str): def get_available_ip( nb: pynetbox.core.api.Api, ip_range: pynetbox.models.ipam.IpRanges ): + """Returns the next available ip address as a Netbox ip object""" logging.info("Finding next available ip") start_address = ip_range.start_address @@ -1194,7 +1195,12 @@ def get_available_ip( return address -def update_ip_dynamo(table, user_email: str, ip_address: str, date_string: str = None): +def update_ip_dynamo( + table: boto3.resources.factory.dynamodb.Table, + user_email: str, + ip_address: str, + date_string: str = None, +): """Creates or updates a ip address with the date string in dynamo""" if date_string is None: @@ -1233,8 +1239,9 @@ def update_ip_dynamo(table, user_email: str, ip_address: str, date_string: str = logging.info("Updated IP on dynamo") -def get_ips_dynamo(table, user_email): +def get_ips_dynamo(table: boto3.resources.factory.dynamodb.Table, user_email: str): """Returns the ip addresses associated with a user""" + response = table.query(KeyConditionExpression=Key("email").eq(user_email)) if "ip_addresses" not in response["Items"][0]: @@ -1245,6 +1252,7 @@ def get_ips_dynamo(table, user_email): def get_dynamo_colab_table(): """Returns dynamo colab table""" + dynamodb = boto3.resource( "dynamodb", region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod From 1d8778e0a7a7df9bd7e5b019f357c188be776029 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 17:30:02 -0400 Subject: [PATCH 15/30] took away bad doc --- features/awx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/awx.py b/features/awx.py index 1ec61ca9..4555fad5 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1196,7 +1196,7 @@ def get_available_ip( def update_ip_dynamo( - table: boto3.resources.factory.dynamodb.Table, + table, user_email: str, ip_address: str, date_string: str = None, @@ -1239,7 +1239,7 @@ def update_ip_dynamo( logging.info("Updated IP on dynamo") -def get_ips_dynamo(table: boto3.resources.factory.dynamodb.Table, user_email: str): +def get_ips_dynamo(table, user_email: str): """Returns the ip addresses associated with a user""" response = table.query(KeyConditionExpression=Key("email").eq(user_email)) From b0067cf542381578f5cab31caf23a90b746d8f22 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 23 Mar 2023 22:45:05 -0400 Subject: [PATCH 16/30] rebased - added list ip function --- bot.py | 6 +++++- features/awx.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bot.py b/bot.py index be08c41b..5a720763 100644 --- a/bot.py +++ b/bot.py @@ -41,7 +41,8 @@ "**CML show IP addresses** > show IP addresses\n", "**CML show server utilization** > show current CPU and Memory usage\n", "**CML stop lab** > stop labs of your choice\n", - "**Request static IP address** > allocates an static ip address for CML\n", + "**Request IP** > allocates a static ip address for CML\n", + "**List my IPs** > lists static IPs allocated to you for CML" "**Create AWS account** > create AWS COLAB account\n", "**Create VPN account** > create an AnyConnect to COLAB VPN account\n", "**Create AWS key** > create aws access key\n", @@ -310,6 +311,9 @@ async def process(self, req: Request): elif self.activity.get("text") == "request ip": await awx.request_ip(self.activity) + elif self.activity.get("text") == "list my ips": + await awx.list_my_ips(self.activity) + elif ( self.activity.get("text")[:3] == "cml" ): # Add searches for cml dialogue here diff --git a/features/awx.py b/features/awx.py index 4555fad5..fbe149c0 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1183,7 +1183,7 @@ def get_available_ip( address = nb.ipam.ip_addresses.get(address=ip) if address is None: # IP not made on netbox - so create - address = nb.ipam.ip_addresses.create(get_ipv4_dict(ip)) + address = nb.ipam.ip_addresses.create(get_ipv4_creation_dict(ip)) break # ip created but not assigned - what we want From 316fb8b28ab04f146a4dad73a2bc917e12ca474c Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 23 Mar 2023 22:52:31 -0400 Subject: [PATCH 17/30] fixed small bugs --- bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot.py b/bot.py index 5a720763..982317cb 100644 --- a/bot.py +++ b/bot.py @@ -42,7 +42,7 @@ "**CML show server utilization** > show current CPU and Memory usage\n", "**CML stop lab** > stop labs of your choice\n", "**Request IP** > allocates a static ip address for CML\n", - "**List my IPs** > lists static IPs allocated to you for CML" + "**List my IPs** > lists static IPs allocated to you for CML\n", "**Create AWS account** > create AWS COLAB account\n", "**Create VPN account** > create an AnyConnect to COLAB VPN account\n", "**Create AWS key** > create aws access key\n", From fea090831b5709195c776b8ab2cc462629de1707 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 17:26:35 -0400 Subject: [PATCH 18/30] rebased and logging --- features/awx.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/features/awx.py b/features/awx.py index fbe149c0..7707d021 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1154,7 +1154,42 @@ async def request_ip(activity): return True -def get_ipv4_dict(ip_address: str): +async def list_my_ips(activity): + """Lists static ip addresses allocated to a user""" + ## APIs + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + table = get_dynamo_colab_table() + + ## Retrieve IPs from database + ip_addresses = get_ips_dynamo(table, activity["sender_email"]) + + # check if ip_address field not there + if not bool(ip_addresses): + message = dict( + text="You do not currently have any allocated IPs", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # get all IPs + markdown = "" + for ip_address, ip_data in ip_addresses.items(): + last_seen = ( + datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) + ).days + markdown += f"{ ip_address }: \n- Last seen: { last_seen } days ago\n\n" + + # send message + message = dict( + text=markdown, + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return True + + +def get_ipv4_creation_dict(ip_address: str): """Helper function for static ip requests""" return {"family": 4, "address": ip_address, "vrf": None} @@ -1245,7 +1280,7 @@ def get_ips_dynamo(table, user_email: str): response = table.query(KeyConditionExpression=Key("email").eq(user_email)) if "ip_addresses" not in response["Items"][0]: - return None + return {} return response["Items"][0]["ip_addresses"] From 896b5e0f4a20223431153d03c487548c55eb6e75 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 17:27:52 -0400 Subject: [PATCH 19/30] repulled --- features/awx.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index 7707d021..40f50328 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1156,6 +1156,9 @@ async def request_ip(activity): async def list_my_ips(activity): """Lists static ip addresses allocated to a user""" + + logging.info("START listing ips") + ## APIs webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) table = get_dynamo_colab_table() @@ -1165,6 +1168,8 @@ async def list_my_ips(activity): # check if ip_address field not there if not bool(ip_addresses): + logging.info("User %s does not have any ips", activity['sender_email'].split('@')[0]) + message = dict( text="You do not currently have any allocated IPs", toPersonId=activity["sender"], @@ -1197,8 +1202,8 @@ def get_ipv4_creation_dict(ip_address: str): def get_available_ip( nb: pynetbox.core.api.Api, ip_range: pynetbox.models.ipam.IpRanges ): - """Returns the next available ip address as a Netbox ip object""" + logging.info("Finding next available ip") start_address = ip_range.start_address mask = start_address[-3:] @@ -1277,6 +1282,8 @@ def update_ip_dynamo( def get_ips_dynamo(table, user_email: str): """Returns the ip addresses associated with a user""" + logging.info("Retrieving IPs") + response = table.query(KeyConditionExpression=Key("email").eq(user_email)) if "ip_addresses" not in response["Items"][0]: From db0168cbdd37900c4cc363800f57850601241aad Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 17:33:17 -0400 Subject: [PATCH 20/30] logging --- features/awx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index 40f50328..e4f8d9d2 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1168,7 +1168,9 @@ async def list_my_ips(activity): # check if ip_address field not there if not bool(ip_addresses): - logging.info("User %s does not have any ips", activity['sender_email'].split('@')[0]) + logging.info( + "User %s does not have any ips", activity["sender_email"].split("@")[0] + ) message = dict( text="You do not currently have any allocated IPs", From c5b90ae85144b4b274e344def1d8c3855d9c791f Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 17:39:40 -0400 Subject: [PATCH 21/30] changed msg look --- features/awx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index e4f8d9d2..eedb1f46 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1185,7 +1185,7 @@ async def list_my_ips(activity): last_seen = ( datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) ).days - markdown += f"{ ip_address }: \n- Last seen: { last_seen } days ago\n\n" + markdown += f"{ ip_address }: \n- Last seen: { last_seen } days ago" # send message message = dict( From 4d82125669b25aef270a08860dcbd020d7bed103 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 17:42:57 -0400 Subject: [PATCH 22/30] changed msg look again --- features/awx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index eedb1f46..32d12cfe 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1185,7 +1185,7 @@ async def list_my_ips(activity): last_seen = ( datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) ).days - markdown += f"{ ip_address }: \n- Last seen: { last_seen } days ago" + markdown += f"{ ip_address }: \n- Last seen: { last_seen } days ago\n" # send message message = dict( From c7f5f9222cb6d13a7c13cb245cc7a91eb7eb4f90 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 20:47:14 -0400 Subject: [PATCH 23/30] changed msg look again --- features/awx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index 32d12cfe..d6f89e5a 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1185,7 +1185,7 @@ async def list_my_ips(activity): last_seen = ( datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) ).days - markdown += f"{ ip_address }: \n- Last seen: { last_seen } days ago\n" + markdown += f" { ip_address }: \n- Last seen: { last_seen } days ago\n" #TODO: Need to break out of the indented - # send message message = dict( From da3b3a3b4c4bd011bca4167b31426f9bef370831 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 20:49:58 -0400 Subject: [PATCH 24/30] changed msg look again --- features/awx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index d6f89e5a..162f0ac6 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1185,7 +1185,7 @@ async def list_my_ips(activity): last_seen = ( datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) ).days - markdown += f" { ip_address }: \n- Last seen: { last_seen } days ago\n" #TODO: Need to break out of the indented - + markdown += f"- { ip_address }: \n - Last seen: { last_seen } days ago\n" # send message message = dict( From 4fddee0c92e7e004d2603c7c92d88319e2fb9ec0 Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 20:54:00 -0400 Subject: [PATCH 25/30] changed msg look again --- features/awx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/awx.py b/features/awx.py index 162f0ac6..318e095b 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1185,7 +1185,7 @@ async def list_my_ips(activity): last_seen = ( datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) ).days - markdown += f"- { ip_address }: \n - Last seen: { last_seen } days ago\n" + markdown += f"{ ip_address }: Last seen { last_seen } days ago\n" # send message message = dict( From 2c5c021758e19d9356a715ba246e263e0a821fdd Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 15:24:44 -0400 Subject: [PATCH 26/30] Added release ips function --- bot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot.py b/bot.py index 982317cb..4a8bc0a4 100644 --- a/bot.py +++ b/bot.py @@ -43,6 +43,7 @@ "**CML stop lab** > stop labs of your choice\n", "**Request IP** > allocates a static ip address for CML\n", "**List my IPs** > lists static IPs allocated to you for CML\n", + "**Release IPs** all | [ips] > return static IPs allocated to you for CML back to pool\n", "**Create AWS account** > create AWS COLAB account\n", "**Create VPN account** > create an AnyConnect to COLAB VPN account\n", "**Create AWS key** > create aws access key\n", @@ -314,6 +315,9 @@ async def process(self, req: Request): elif self.activity.get("text") == "list my ips": await awx.list_my_ips(self.activity) + elif self.activity.get("text") == "release ips": + await awx.release_ips(self.activity) + elif ( self.activity.get("text")[:3] == "cml" ): # Add searches for cml dialogue here From 5ef45cc0de3f8335897a111044826ef12ca941ad Mon Sep 17 00:00:00 2001 From: KJStick Date: Thu, 15 Jun 2023 21:17:41 -0400 Subject: [PATCH 27/30] rebased --- features/awx.py | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/features/awx.py b/features/awx.py index 318e095b..b907a669 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1196,8 +1196,99 @@ async def list_my_ips(activity): return True +async def release_ips(activity): + """Returns a list of IPs back to the static pool""" + nb_url = str(CONFIG.NETBOX_URL) + nb_token = str(CONFIG.NETBOX_TOKEN) + username = activity["sender_email"].split("@")[0] + non_returned_ips = [] + returned_ips = [] + + ## APIs + nb = pynetbox.api(nb_url, nb_token) + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + table = get_dynamo_colab_table() + + # Retrieve IPs from database + all_ip_addresses = get_ips_dynamo(table, activity['sender_email']) + + # check if ip_address field not there + if not bool(all_ip_addresses): + message = dict( + text="You do not currently have any allocated IPs", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # get ips indicated by user to return + #ips = activity['ips'] + ips = 'all' + + if ips != 'all': # Get specific IPs indicated by the user + ip_addresses = {ip: all_ip_addresses[ip] for ip in ips if ip in all_ip_addresses} + non_returned_ips = [ip for ip in ips if ip not in ip_addresses] + + else: # Return all IPs + ip_addresses = all_ip_addresses + + # Return IPs + for ip_address in ip_addresses: + ## Return in Netbox + address = nb.ipam.ip_addresses.get(address=ip_address) + if address is None or username not in address.description: + # IP not made on netbox - ERROR, or not assigned to this user + non_returned_ips.append(ip_address) + continue + + # unassign to user on netbox + address.description = "" + address.status = "active" + address.save() + + ## Return in Database + table.update_item( + Key={"email": activity["sender_email"]}, + UpdateExpression="REMOVE #ip_addresses.#ip_address", + ExpressionAttributeNames={ + "#ip_addresses": "ip_addresses", + "#ip_address": str(ip_address), + }, + ) + + returned_ips.append(ip_address) + + # Message user if bad ip - either not valid or they do not own it + if non_returned_ips: + markdown = "**IPS NOT RETURNED**\n- **Reason:** Not valid ip or not allocated to you" + for ip in non_returned_ips: + markdown += f"\n- { ip }" + + message = dict( + text=markdown, + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + + + if returned_ips: + markdown = "**IPS RETURNED**\n" + "\n".join(f"- {ip}" for ip in returned_ips) + + else: # No ips returned + markdown = "There are no valid allocated IPs to return" + + message = dict( + text=markdown, + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + + return True + + def get_ipv4_creation_dict(ip_address: str): """Helper function for static ip requests""" + return {"family": 4, "address": ip_address, "vrf": None} From 2aa8940f6e997d9e48c3a6692859cb9ac34784d2 Mon Sep 17 00:00:00 2001 From: KJStick Date: Fri, 16 Jun 2023 15:27:41 -0400 Subject: [PATCH 28/30] simplified --- features/awx.py | 65 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/features/awx.py b/features/awx.py index b907a669..b069c46d 100644 --- a/features/awx.py +++ b/features/awx.py @@ -31,6 +31,9 @@ find_user_message = "Cannot find user" PRE_CODE_SNIPPET = "
"
 AFTER_CODE_SNIPPET = "
" +MAIN_IP_TAG = "#ip_addresses" +MAIN_IP_TAG_VAL = "ip_addresses" +IP_TAG = "#ip_address" mongo_url = ( "mongodb://" @@ -1136,7 +1139,7 @@ async def request_ip(activity): return False # assign to user on netbox - address.description = username + address.description += " " + username address.status = "reserved" address.save() logging.info("Saved IP on netbox") @@ -1198,6 +1201,8 @@ async def list_my_ips(activity): async def release_ips(activity): """Returns a list of IPs back to the static pool""" + + logging.info("Start release ips") nb_url = str(CONFIG.NETBOX_URL) nb_token = str(CONFIG.NETBOX_TOKEN) username = activity["sender_email"].split("@")[0] @@ -1210,10 +1215,11 @@ async def release_ips(activity): table = get_dynamo_colab_table() # Retrieve IPs from database - all_ip_addresses = get_ips_dynamo(table, activity['sender_email']) + all_ip_addresses = get_ips_dynamo(table, activity["sender_email"]) # check if ip_address field not there if not bool(all_ip_addresses): + logging.info("User has no ips") message = dict( text="You do not currently have any allocated IPs", toPersonId=activity["sender"], @@ -1222,11 +1228,13 @@ async def release_ips(activity): return False # get ips indicated by user to return - #ips = activity['ips'] - ips = 'all' + # ips = activity['ips'] + ips = "all" - if ips != 'all': # Get specific IPs indicated by the user - ip_addresses = {ip: all_ip_addresses[ip] for ip in ips if ip in all_ip_addresses} + if ips != "all": # Get specific IPs indicated by the user + ip_addresses = { + ip: all_ip_addresses[ip] for ip in ips if ip in all_ip_addresses + } non_returned_ips = [ip for ip in ips if ip not in ip_addresses] else: # Return all IPs @@ -1242,25 +1250,20 @@ async def release_ips(activity): continue # unassign to user on netbox - address.description = "" + address.description.replace(f" {username}", "") address.status = "active" address.save() - ## Return in Database - table.update_item( - Key={"email": activity["sender_email"]}, - UpdateExpression="REMOVE #ip_addresses.#ip_address", - ExpressionAttributeNames={ - "#ip_addresses": "ip_addresses", - "#ip_address": str(ip_address), - }, - ) + remove_ip_dynamo(table, activity["sender_email"], ip_address) returned_ips.append(ip_address) + logging.info("IPs released") # Message user if bad ip - either not valid or they do not own it if non_returned_ips: - markdown = "**IPS NOT RETURNED**\n- **Reason:** Not valid ip or not allocated to you" + markdown = ( + "**IPs Not Returned**\n- **Reason:** Not valid ip or not allocated to you" + ) for ip in non_returned_ips: markdown += f"\n- { ip }" @@ -1270,11 +1273,10 @@ async def release_ips(activity): ) await webex.post_message_to_webex(message) - if returned_ips: - markdown = "**IPS RETURNED**\n" + "\n".join(f"- {ip}" for ip in returned_ips) + markdown = "**IPs Returned**\n" + "\n".join(f"- {ip}" for ip in returned_ips) - else: # No ips returned + else: # No ips returned markdown = "There are no valid allocated IPs to return" message = dict( @@ -1336,6 +1338,8 @@ def update_ip_dynamo( ): """Creates or updates a ip address with the date string in dynamo""" + logging.info("Updating ip %s", str(ip_address)) + if date_string is None: date_string = str(int(datetime.timestamp(datetime.now()))) @@ -1350,7 +1354,7 @@ def update_ip_dynamo( table.update_item( Key={"email": user_email}, UpdateExpression="SET #ip_addresses= :value", - ExpressionAttributeNames={"#ip_addresses": "ip_addresses"}, + ExpressionAttributeNames={MAIN_IP_TAG: MAIN_IP_TAG_VAL}, ExpressionAttributeValues={":value": {}}, ) @@ -1359,8 +1363,8 @@ def update_ip_dynamo( Key={"email": user_email}, UpdateExpression="SET #ip_addresses.#ip_address= :ip_data", ExpressionAttributeNames={ - "#ip_addresses": "ip_addresses", - "#ip_address": str(ip_address), + MAIN_IP_TAG: MAIN_IP_TAG_VAL, + IP_TAG: str(ip_address), }, ExpressionAttributeValues={ ":ip_data": { @@ -1372,6 +1376,21 @@ def update_ip_dynamo( logging.info("Updated IP on dynamo") +def remove_ip_dynamo(table, user_email: str, ip_address: str): + """Removes an IP address from a user's database""" + + logging.info("Removing ip %s from database", ip_address) + + table.update_item( + Key={"email": user_email}, + UpdateExpression="REMOVE #ip_addresses.#ip_address", + ExpressionAttributeNames={ + MAIN_IP_TAG: MAIN_IP_TAG_VAL, + IP_TAG: ip_address, + }, + ) + + def get_ips_dynamo(table, user_email: str): """Returns the ip addresses associated with a user""" From a6ed12684c1cee9098f51d9e82b26e16fb4ce648 Mon Sep 17 00:00:00 2001 From: KJStick Date: Sat, 17 Jun 2023 20:53:25 -0400 Subject: [PATCH 29/30] added regex matching --- bot.py | 11 +++++++++-- features/awx.py | 15 ++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/bot.py b/bot.py index 4a8bc0a4..354c7fdd 100644 --- a/bot.py +++ b/bot.py @@ -6,6 +6,7 @@ import json import time import logging +import re import aiohttp import pymongo import urllib3 @@ -58,6 +59,8 @@ "**help** > display available commands\n", ] +release_ips_pattern = r"release ips (?:all|\d{1,3}(?:\.\d{1,3}){3}(?:\s+\d{1,3}(?:\.\d{1,3}){3})*)" # command = release ips [all | ips] + class COLABot: def __init__(self, webex_bot_token=None, webex_client_signing_secret=None): @@ -309,13 +312,17 @@ async def process(self, req: Request): elif self.activity.get("text") == "delete accounts": await awx.delete_accounts(self.activity) - elif self.activity.get("text") == "request ip": + elif re.findall( + release_ips_pattern, self.activity.get("text") + ): # command = release ips [all | ips] + matches = re.findall(release_ips_pattern, self.activity.get("text")) + self.activity["ips_to_release"] = matches[0].split(" ")[2:] await awx.request_ip(self.activity) elif self.activity.get("text") == "list my ips": await awx.list_my_ips(self.activity) - elif self.activity.get("text") == "release ips": + elif self.activity.get("original_text") == "release ips": await awx.release_ips(self.activity) elif ( diff --git a/features/awx.py b/features/awx.py index b069c46d..eca1a1ba 100644 --- a/features/awx.py +++ b/features/awx.py @@ -1183,12 +1183,14 @@ async def list_my_ips(activity): return False # get all IPs + length = max(len(ip) for ip in ip_addresses) markdown = "" for ip_address, ip_data in ip_addresses.items(): last_seen = ( datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) ).days - markdown += f"{ ip_address }: Last seen { last_seen } days ago\n" + formatted_ip = f"{ip_address:<{length}}" # format so all the same length + markdown += f"{ formatted_ip }: Last seen { last_seen } days ago\n" # send message message = dict( @@ -1228,18 +1230,17 @@ async def release_ips(activity): return False # get ips indicated by user to return - # ips = activity['ips'] - ips = "all" + ips = activity["ips_to_release"] - if ips != "all": # Get specific IPs indicated by the user + if "all" in ips: # Return all IPs + ip_addresses = all_ip_addresses + + else: # Get specific IPs indicated by the user ip_addresses = { ip: all_ip_addresses[ip] for ip in ips if ip in all_ip_addresses } non_returned_ips = [ip for ip in ips if ip not in ip_addresses] - else: # Return all IPs - ip_addresses = all_ip_addresses - # Return IPs for ip_address in ip_addresses: ## Return in Netbox From 52724ed3b65d93b7afa47b645c930a1a6814ea13 Mon Sep 17 00:00:00 2001 From: KJStick Date: Sat, 17 Jun 2023 21:03:15 -0400 Subject: [PATCH 30/30] updated regex matching --- bot.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/bot.py b/bot.py index 354c7fdd..1c648ecc 100644 --- a/bot.py +++ b/bot.py @@ -59,7 +59,7 @@ "**help** > display available commands\n", ] -release_ips_pattern = r"release ips (?:all|\d{1,3}(?:\.\d{1,3}){3}(?:\s+\d{1,3}(?:\.\d{1,3}){3})*)" # command = release ips [all | ips] +release_ips_pattern = r"release ips (?:all|\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?(?:\s+\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)*)" # command = release ips [all | ips] class COLABot: @@ -312,17 +312,19 @@ async def process(self, req: Request): elif self.activity.get("text") == "delete accounts": await awx.delete_accounts(self.activity) - elif re.findall( - release_ips_pattern, self.activity.get("text") - ): # command = release ips [all | ips] - matches = re.findall(release_ips_pattern, self.activity.get("text")) - self.activity["ips_to_release"] = matches[0].split(" ")[2:] + elif self.activity.get("text") == "request ip": await awx.request_ip(self.activity) elif self.activity.get("text") == "list my ips": await awx.list_my_ips(self.activity) - elif self.activity.get("original_text") == "release ips": + elif re.findall( + release_ips_pattern, self.activity.get("original_text") + ): # command = release ips [all | ips] + matches = re.findall( + release_ips_pattern, self.activity.get("original_text") + ) + self.activity["ips_to_release"] = matches[0].split(" ")[2:] await awx.release_ips(self.activity) elif (