diff --git a/.github/workflows/ci_dev.yaml b/.github/workflows/ci_dev.yaml index ac33d650..79ce0c85 100644 --- a/.github/workflows/ci_dev.yaml +++ b/.github/workflows/ci_dev.yaml @@ -86,7 +86,7 @@ jobs: MONGO_DB_ROOT_PASSWORD: ${{ secrets.MONGO_DB_ROOT_PASSWORD }} AWX_PASSWORD: ${{ secrets.AWX_PASSWORD }} NLP_SECRET: ${{ secrets.NLP_SECRET }} - AWS_PASSWORD: ${{ secrets.AWS_PASSWORD }} + AWS_PASSWORD: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} BOT_ID_DEV: ${{ secrets.BOT_ID_DEV }} BOT_NAME_DEV: ${{ secrets.BOT_NAME_DEV }} AUTHORIZED_ROOMS_DEV: ${{ secrets.AUTHORIZED_ROOMS_DEV }} @@ -100,11 +100,11 @@ jobs: NLP_SERVER_DEV: ${{ secrets.NLP_SERVER_DEV }} VCENTER_SERVER: ${{ secrets.VCENTER_SERVER }} ADMINISTRATORS_DEV: ${{ secrets.ADMINISTRATORS_DEV }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_REGION_COLAB: ${{ secrets.AWS_REGION_COLAB }} - AWS_ACCESS_KEY_ID_COLAB: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} - AWS_PASSWORD_COLAB: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} COLABOT_SECRET: ${{ secrets.COLABOT_SECRET }} + DYNAMO_TABLE: ${{ secrets.DYNAMO_TABLE_DEV }} + NETBOX_URL: ${{ secrets.NETBOX_URL_DEV }} + NETBOX_TOKEN: ${{ secrets.NETBOX_TOKEN_DEV }} run: python3 process-j2.py - name: Apply and rollout diff --git a/.github/workflows/ci_prod.yaml b/.github/workflows/ci_prod.yaml index f5cd6c75..00095f56 100644 --- a/.github/workflows/ci_prod.yaml +++ b/.github/workflows/ci_prod.yaml @@ -101,10 +101,10 @@ jobs: VCENTER_SERVER: ${{ secrets.VCENTER_SERVER }} ADMINISTRATORS_PROD: ${{ secrets.ADMINISTRATORS_PROD }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_REGION_COLAB: ${{ secrets.AWS_REGION_COLAB }} - AWS_ACCESS_KEY_ID_COLAB: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} - AWS_PASSWORD_COLAB: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} COLABOT_SECRET: ${{ secrets.COLABOT_SECRET }} + DYNAMO_TABLE: ${{ secrets.DYNAMO_TABLE_PROD }} + NETBOX_URL: ${{ secrets.NETBOX_URL_PROD }} + NETBOX_TOKEN: ${{ secrets.NETBOX_TOKEN_PROD }} run: python3 process-j2.py - name: Apply and rollout diff --git a/bot.py b/bot.py index d23a104a..1c648ecc 100644 --- a/bot.py +++ b/bot.py @@ -6,6 +6,7 @@ import json import time import logging +import re import aiohttp import pymongo import urllib3 @@ -41,6 +42,9 @@ "**CML show IP addresses** > show IP addresses\n", "**CML show server utilization** > show current CPU and Memory usage\n", "**CML stop lab** > stop labs of your choice\n", + "**Request IP** > allocates a static ip address for CML\n", + "**List my IPs** > lists static IPs allocated to you for CML\n", + "**Release IPs** all | [ips] > return static IPs allocated to you for CML back to pool\n", "**Create AWS account** > create AWS COLAB account\n", "**Create VPN account** > create an AnyConnect to COLAB VPN account\n", "**Create AWS key** > create aws access key\n", @@ -55,6 +59,8 @@ "**help** > display available commands\n", ] +release_ips_pattern = r"release ips (?:all|\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?(?:\s+\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)*)" # command = release ips [all | ips] + class COLABot: def __init__(self, webex_bot_token=None, webex_client_signing_secret=None): @@ -306,6 +312,21 @@ async def process(self, req: Request): elif self.activity.get("text") == "delete accounts": await awx.delete_accounts(self.activity) + elif self.activity.get("text") == "request ip": + await awx.request_ip(self.activity) + + elif self.activity.get("text") == "list my ips": + await awx.list_my_ips(self.activity) + + elif re.findall( + release_ips_pattern, self.activity.get("original_text") + ): # command = release ips [all | ips] + matches = re.findall( + release_ips_pattern, self.activity.get("original_text") + ) + self.activity["ips_to_release"] = matches[0].split(" ")[2:] + await awx.release_ips(self.activity) + elif ( self.activity.get("text")[:3] == "cml" ): # Add searches for cml dialogue here diff --git a/colabot-manifest-dev.yaml.j2 b/colabot-manifest-dev.yaml.j2 index dd8487f9..62480514 100644 --- a/colabot-manifest-dev.yaml.j2 +++ b/colabot-manifest-dev.yaml.j2 @@ -110,7 +110,7 @@ spec: - name: ADMINISTRATORS value: {{ ADMINISTRATORS_DEV }} - name: AWS_ACCESS_KEY_ID - value: {{ AWS_ACCESS_KEY_ID_COLAB }} + value: {{ AWS_ACCESS_KEY_ID }} - name: AWS_REGION value: 'us-east-1' - name: AWS_SECRET_ACCESS_KEY @@ -121,11 +121,11 @@ spec: - name: COLABOT_SECRET value: {{ COLABOT_SECRET }} - name: AWS_DYNAMO_TABLE - value: {{ DYNAMO_TABLE_DEV }} + value: {{ DYNAMO_TABLE }} - name: NETBOX_URL - value: {{ NETBOX_URL_DEV }} + value: {{ NETBOX_URL }} - name: NETBOX_TOKEN - value: {{ NETBOX_TOKEN_DEV }} + value: {{ NETBOX_TOKEN }} --- diff --git a/colabot-manifest-prod.yaml.j2 b/colabot-manifest-prod.yaml.j2 index d3a5d5f4..79b49cba 100644 --- a/colabot-manifest-prod.yaml.j2 +++ b/colabot-manifest-prod.yaml.j2 @@ -114,11 +114,11 @@ spec: - name: COLABOT_SECRET value: {{ COLABOT_SECRET }} - name: AWS_DYNAMO_TABLE - value: {{ DYNAMO_TABLE_PROD }} + value: {{ DYNAMO_TABLE }} - name: NETBOX_URL - value: {{ NETBOX_URL_PROD }} + value: {{ NETBOX_URL }} - name: NETBOX_TOKEN - value: {{ NETBOX_TOKEN_PROD }} + value: {{ NETBOX_TOKEN }} --- diff --git a/colabot-secrets-dev.yaml.j2 b/colabot-secrets-dev.yaml.j2 index d4cd6226..3e5f31de 100644 --- a/colabot-secrets-dev.yaml.j2 +++ b/colabot-secrets-dev.yaml.j2 @@ -13,4 +13,4 @@ stringData: mongo_initb_root_password: {{ MONGO_DB_ROOT_PASSWORD }} awx_password: {{ AWX_PASSWORD }} nlp_secret: {{ NLP_SECRET }} - aws_password: {{ AWS_PASSWORD_COLAB }} + aws_password: {{ AWS_PASSWORD }} diff --git a/config.py b/config.py index 27ceda12..d99ca103 100644 --- a/config.py +++ b/config.py @@ -48,4 +48,3 @@ class DefaultConfig: COLABOT_CYPHER = os.environ.get("COLABOT_SECRET") NETBOX_URL = os.environ.get("NETBOX_URL") NETBOX_TOKEN = os.environ.get("NETBOX_TOKEN") - diff --git a/features/awx.py b/features/awx.py index 8cb8ddf8..eca1a1ba 100644 --- a/features/awx.py +++ b/features/awx.py @@ -5,12 +5,14 @@ import re import tempfile from datetime import datetime, date +import ipaddress from cryptography.fernet import Fernet import aiohttp import pymongo import urllib3 import boto3 -from boto3.dynamodb.conditions import Key +from boto3.dynamodb.conditions import Key, Attr +import pynetbox import yaml from virl2_client import ClientLibrary from jinja2 import Template @@ -29,6 +31,9 @@ find_user_message = "Cannot find user" PRE_CODE_SNIPPET = "
"
 AFTER_CODE_SNIPPET = "
" +MAIN_IP_TAG = "#ip_addresses" +MAIN_IP_TAG_VAL = "ip_addresses" +IP_TAG = "#ip_address" mongo_url = ( "mongodb://" @@ -640,16 +645,7 @@ async def handle_labbing_card(activity): labs_to_save = [] labs_to_delete = [] - dynamodb = boto3.resource( - "dynamodb", - region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod - aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, - aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, - ) - - table = dynamodb.Table( - "colab_directory" # Table Name - ) # TODO remove dev extension when pushing to prod + table = get_dynamo_colab_table() cml_server = CONFIG.SERVER_LIST.split(",")[0] user_and_domain = user_email.split("@") @@ -1083,3 +1079,342 @@ async def get_iam_user(iam_username, iam=None): return return user + + +async def request_ip(activity): + """Allocates a static ip from netbox for cml lab use""" + + logging.info("Start Request IP") + ip_limit = 10 + nb_url = str(CONFIG.NETBOX_URL) + nb_token = str(CONFIG.NETBOX_TOKEN) + username = activity["sender_email"].split("@")[0] + + ## APIs + nb = pynetbox.api(nb_url, nb_token) + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + table = get_dynamo_colab_table() + + ## make sure user under ip limit + ip_addresses = get_ips_dynamo(table, activity["sender_email"]) + + if len(ip_addresses) >= ip_limit: + logging.info("User %s has reached limit of %s", username, str(ip_limit)) + message = dict( + text=f"You have reached the limit of { ip_limit } reserved ip addresses", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # Find static ip pool on netbox + ip_ranges = nb.ipam.ip_ranges.all() + ip_range = None + for ip_range in ip_ranges: + if "static ips" in ip_range["description"].lower(): + break + ip_range = None + + if ip_range is None: + message = dict( + text="No IP pool could be found on Netbox", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + logging.info("Found IP range %s", str(ip_range)) + + # ip_type = ip_range.family.label + # check if ipv4 or 6 - below assumes 4 + + address = get_available_ip(nb, ip_range) + logging.info("Got IP address %s", str(address)) + + if address is None: + message = dict( + text="There are no more ips available", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # assign to user on netbox + address.description += " " + username + address.status = "reserved" + address.save() + logging.info("Saved IP on netbox") + + ## insert ip into database + update_ip_dynamo(table, activity["sender_email"], address) + + # message user + message = dict( + text=f"New static IP Address assigned: { str(address) }", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + + return True + + +async def list_my_ips(activity): + """Lists static ip addresses allocated to a user""" + + logging.info("START listing ips") + + ## APIs + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + table = get_dynamo_colab_table() + + ## Retrieve IPs from database + ip_addresses = get_ips_dynamo(table, activity["sender_email"]) + + # check if ip_address field not there + if not bool(ip_addresses): + logging.info( + "User %s does not have any ips", activity["sender_email"].split("@")[0] + ) + + message = dict( + text="You do not currently have any allocated IPs", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # get all IPs + length = max(len(ip) for ip in ip_addresses) + markdown = "" + for ip_address, ip_data in ip_addresses.items(): + last_seen = ( + datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) + ).days + formatted_ip = f"{ip_address:<{length}}" # format so all the same length + markdown += f"{ formatted_ip }: Last seen { last_seen } days ago\n" + + # send message + message = dict( + text=markdown, + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return True + + +async def release_ips(activity): + """Returns a list of IPs back to the static pool""" + + logging.info("Start release ips") + nb_url = str(CONFIG.NETBOX_URL) + nb_token = str(CONFIG.NETBOX_TOKEN) + username = activity["sender_email"].split("@")[0] + non_returned_ips = [] + returned_ips = [] + + ## APIs + nb = pynetbox.api(nb_url, nb_token) + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + table = get_dynamo_colab_table() + + # Retrieve IPs from database + all_ip_addresses = get_ips_dynamo(table, activity["sender_email"]) + + # check if ip_address field not there + if not bool(all_ip_addresses): + logging.info("User has no ips") + message = dict( + text="You do not currently have any allocated IPs", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # get ips indicated by user to return + ips = activity["ips_to_release"] + + if "all" in ips: # Return all IPs + ip_addresses = all_ip_addresses + + else: # Get specific IPs indicated by the user + ip_addresses = { + ip: all_ip_addresses[ip] for ip in ips if ip in all_ip_addresses + } + non_returned_ips = [ip for ip in ips if ip not in ip_addresses] + + # Return IPs + for ip_address in ip_addresses: + ## Return in Netbox + address = nb.ipam.ip_addresses.get(address=ip_address) + if address is None or username not in address.description: + # IP not made on netbox - ERROR, or not assigned to this user + non_returned_ips.append(ip_address) + continue + + # unassign to user on netbox + address.description.replace(f" {username}", "") + address.status = "active" + address.save() + + remove_ip_dynamo(table, activity["sender_email"], ip_address) + + returned_ips.append(ip_address) + logging.info("IPs released") + + # Message user if bad ip - either not valid or they do not own it + if non_returned_ips: + markdown = ( + "**IPs Not Returned**\n- **Reason:** Not valid ip or not allocated to you" + ) + for ip in non_returned_ips: + markdown += f"\n- { ip }" + + message = dict( + text=markdown, + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + + if returned_ips: + markdown = "**IPs Returned**\n" + "\n".join(f"- {ip}" for ip in returned_ips) + + else: # No ips returned + markdown = "There are no valid allocated IPs to return" + + message = dict( + text=markdown, + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + + return True + + +def get_ipv4_creation_dict(ip_address: str): + """Helper function for static ip requests""" + + return {"family": 4, "address": ip_address, "vrf": None} + + +def get_available_ip( + nb: pynetbox.core.api.Api, ip_range: pynetbox.models.ipam.IpRanges +): + """Returns the next available ip address as a Netbox ip object""" + + logging.info("Finding next available ip") + start_address = ip_range.start_address + mask = start_address[-3:] + net = ipaddress.ip_network(start_address, False) + + # Find first available ip + valid_ip = False + for ip in net: + ip = str(ip) + mask + + # make sure to only check at start of range, not start of net + if ip == start_address: + valid_ip = True + if not valid_ip: + continue + + address = nb.ipam.ip_addresses.get(address=ip) + if address is None: + # IP not made on netbox - so create + address = nb.ipam.ip_addresses.create(get_ipv4_creation_dict(ip)) + break + + # ip created but not assigned - what we want + if address.description == "" and address.status == "active": + break + + address = None + + return address + + +def update_ip_dynamo( + table, + user_email: str, + ip_address: str, + date_string: str = None, +): + """Creates or updates a ip address with the date string in dynamo""" + + logging.info("Updating ip %s", str(ip_address)) + + if date_string is None: + date_string = str(int(datetime.timestamp(datetime.now()))) + + # check to see if ip field already there + response = table.query( + KeyConditionExpression=Key("email").eq(user_email), + FilterExpression=Attr("ip_addresses").exists(), + ) + + # create ip field map if it doesn't exist + if response["Count"] == 0: + table.update_item( + Key={"email": user_email}, + UpdateExpression="SET #ip_addresses= :value", + ExpressionAttributeNames={MAIN_IP_TAG: MAIN_IP_TAG_VAL}, + ExpressionAttributeValues={":value": {}}, + ) + + # insert new ip + table.update_item( + Key={"email": user_email}, + UpdateExpression="SET #ip_addresses.#ip_address= :ip_data", + ExpressionAttributeNames={ + MAIN_IP_TAG: MAIN_IP_TAG_VAL, + IP_TAG: str(ip_address), + }, + ExpressionAttributeValues={ + ":ip_data": { + "date_last_used": date_string, + } + }, + ) + + logging.info("Updated IP on dynamo") + + +def remove_ip_dynamo(table, user_email: str, ip_address: str): + """Removes an IP address from a user's database""" + + logging.info("Removing ip %s from database", ip_address) + + table.update_item( + Key={"email": user_email}, + UpdateExpression="REMOVE #ip_addresses.#ip_address", + ExpressionAttributeNames={ + MAIN_IP_TAG: MAIN_IP_TAG_VAL, + IP_TAG: ip_address, + }, + ) + + +def get_ips_dynamo(table, user_email: str): + """Returns the ip addresses associated with a user""" + + logging.info("Retrieving IPs") + + response = table.query(KeyConditionExpression=Key("email").eq(user_email)) + + if "ip_addresses" not in response["Items"][0]: + return {} + + return response["Items"][0]["ip_addresses"] + + +def get_dynamo_colab_table(): + """Returns dynamo colab table""" + + dynamodb = boto3.resource( + "dynamodb", + region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod + aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, + aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, + ) + + table = dynamodb.Table(CONFIG.AWS_DYNAMO_TABLE) # Table Name + + return table diff --git a/requirements.txt b/requirements.txt index 9460abd1..5a92ac4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ webexteamssdk==1.6 virl2_client==2.4.0 PyYAML==5.4 cryptography==39.0.1 +ipaddress==1.0.23 +pynetbox==7.0.1