diff --git a/src/azure-cli-core/azure/cli/core/__init__.py b/src/azure-cli-core/azure/cli/core/__init__.py index fc4dcae82c1..4fa55c0bfd4 100644 --- a/src/azure-cli-core/azure/cli/core/__init__.py +++ b/src/azure-cli-core/azure/cli/core/__init__.py @@ -9,9 +9,7 @@ import os import sys import json -import timeit -import concurrent.futures -from concurrent.futures import ThreadPoolExecutor +import time from knack.cli import CLI from knack.commands import CLICommandsLoader @@ -322,14 +320,14 @@ def _update_command_table_from_modules(args, command_modules=None): except ImportError as e: logger.warning(e) - start_time = timeit.default_timer() + start_time = time.perf_counter() logger.debug("Loading command modules...") results = self._load_modules(args, command_modules) count, cumulative_group_count, cumulative_command_count = \ self._process_results_with_timing(results) - total_elapsed_time = timeit.default_timer() - start_time + total_elapsed_time = time.perf_counter() - start_time # Summary line logger.debug(self.item_format_string, "Total ({})".format(count), total_elapsed_time, @@ -404,7 +402,7 @@ def _filter_modname(extensions): # Add to the map. This needs to happen before we load commands as registering a command # from an extension requires this map to be up-to-date. # self._mod_to_ext_map[ext_mod] = ext_name - start_time = timeit.default_timer() + start_time = time.perf_counter() extension_command_table, extension_group_table, extension_command_loader = \ _load_extension_command_loader(self, args, ext_mod) import_extension_breaking_changes(ext_mod) @@ -427,7 +425,7 @@ def _filter_modname(extensions): self.command_table.update(extension_command_table) self.command_group_table.update(extension_group_table) - elapsed_time = timeit.default_timer() - start_time + elapsed_time = time.perf_counter() - start_time logger.debug(self.item_ext_format_string, ext_name, elapsed_time, len(extension_group_table), len(extension_command_table), ext_dir) @@ -667,6 +665,8 @@ def load_arguments(self, command=None): def _load_modules(self, args, command_modules): """Load command modules using ThreadPoolExecutor with timeout protection.""" + import concurrent.futures + from concurrent.futures import ThreadPoolExecutor from azure.cli.core.commands import BLOCKED_MODS results = [] @@ -708,10 +708,10 @@ def _load_single_module(self, mod, args): from azure.cli.core.commands import _load_module_command_loader import traceback try: - start_time = timeit.default_timer() + start_time = time.perf_counter() module_command_table, module_group_table, command_loader = _load_module_command_loader(self, args, mod) import_module_breaking_changes(mod) - elapsed_time = timeit.default_timer() - start_time + elapsed_time = time.perf_counter() - start_time return ModuleLoadResult(mod, module_command_table, module_group_table, elapsed_time, command_loader=command_loader) except Exception as ex: # pylint: disable=broad-except tb_str = traceback.format_exc() @@ -1227,7 +1227,7 @@ def update(self, command_table): :param command_table: The command table built by azure.cli.core.MainCommandsLoader.load_command_table """ - start_time = timeit.default_timer() + start_time = time.perf_counter() self.INDEX[self._COMMAND_INDEX_VERSION] = __version__ self.INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = self.cloud_profile from collections import defaultdict @@ -1242,7 +1242,7 @@ def update(self, command_table): if module_name not in index[top_command]: index[top_command].append(module_name) - elapsed_time = timeit.default_timer() - start_time + elapsed_time = time.perf_counter() - start_time self.INDEX[self._COMMAND_INDEX] = index self.update_extension_index(command_table) diff --git a/src/azure-cli-core/azure/cli/core/_help_loaders.py b/src/azure-cli-core/azure/cli/core/_help_loaders.py index 7be465e1b97..d97fe3c3833 100644 --- a/src/azure-cli-core/azure/cli/core/_help_loaders.py +++ b/src/azure-cli-core/azure/cli/core/_help_loaders.py @@ -11,8 +11,6 @@ from knack.util import CLIError from knack.log import get_logger -import yaml - logger = get_logger(__name__) try: @@ -132,6 +130,8 @@ def _get_yaml_help_files_list(nouns, cmd_loader_map_ref): @staticmethod def _parse_yaml_from_string(text, help_file_path): + import yaml + dir_name, base_name = os.path.split(help_file_path) pretty_file_path = os.path.join(os.path.basename(dir_name), base_name) diff --git a/src/azure-cli-core/azure/cli/core/aaz/_client.py b/src/azure-cli-core/azure/cli/core/aaz/_client.py index 4d39bedaab0..0455bc7679c 100644 --- a/src/azure-cli-core/azure/cli/core/aaz/_client.py +++ b/src/azure-cli-core/azure/cli/core/aaz/_client.py @@ -8,7 +8,6 @@ from azure.core.polling.base_polling import LocationPolling, StatusCheckPolling from abc import abstractmethod -from ._poller import AAZNoPolling, AAZBasePolling from azure.cli.core.cloud import (CloudEndpointNotSetException, CloudSuffixNotSetException, CloudNameEnum as _CloudNameEnum) @@ -111,6 +110,7 @@ def send_request(self, request, stream=False, **kwargs): # pylint: disable=argu def build_lro_polling(self, no_wait, initial_session, deserialization_callback, error_callback, lro_options=None, path_format_arguments=None): from azure.core.polling.base_polling import OperationResourcePolling + from ._poller import AAZNoPolling, AAZBasePolling if no_wait == True: # noqa: E712, pylint: disable=singleton-comparison polling = AAZNoPolling() else: @@ -233,6 +233,7 @@ def _build_per_call_policies(cls, ctx, **kwargs): def build_lro_polling(self, no_wait, initial_session, deserialization_callback, error_callback, lro_options=None, path_format_arguments=None): from azure.mgmt.core.polling.arm_polling import AzureAsyncOperationPolling, BodyContentPolling + from ._poller import AAZNoPolling, AAZBasePolling if no_wait == True: # noqa: E712, pylint: disable=singleton-comparison polling = AAZNoPolling() else: diff --git a/src/azure-cli-core/azure/cli/core/aaz/_command.py b/src/azure-cli-core/azure/cli/core/aaz/_command.py index 577d8a7cf45..e47c461315e 100644 --- a/src/azure-cli-core/azure/cli/core/aaz/_command.py +++ b/src/azure-cli-core/azure/cli/core/aaz/_command.py @@ -22,7 +22,6 @@ from ._base import AAZUndefined, AAZBaseValue from ._field_type import AAZObjectType from ._paging import AAZPaged -from ._poller import AAZLROPoller from ._command_ctx import AAZCommandCtx from .exceptions import AAZUnknownFieldError, AAZUnregisteredArg from .utils import get_aaz_profile_module_name @@ -235,6 +234,7 @@ def processor(schema, result): def build_lro_poller(self, executor, extract_result): """ Build AAZLROPoller instance to support long running operation """ + from ._poller import AAZLROPoller polling_generator = executor() if self.ctx.lro_no_wait: # run until yield the first polling diff --git a/src/azure-cli-core/azure/cli/core/commands/__init__.py b/src/azure-cli-core/azure/cli/core/commands/__init__.py index a9dc9cfbc43..2700595662a 100644 --- a/src/azure-cli-core/azure/cli/core/commands/__init__.py +++ b/src/azure-cli-core/azure/cli/core/commands/__init__.py @@ -29,7 +29,6 @@ get_command_type_kwarg, read_file_content, get_arg_list, poller_classes) from azure.cli.core.local_context import LocalContextAction from azure.cli.core import telemetry -from azure.cli.core.commands.progress import IndeterminateProgressBar from knack.arguments import CLICommandArgument from knack.commands import CLICommand, CommandGroup, PREVIEW_EXPERIMENTAL_CONFLICT_ERROR @@ -1035,10 +1034,20 @@ def __init__(self, cli_ctx, start_msg='', finish_msg='', poller_done_interval_ms self.deploy_dict = {} self.last_progress_report = datetime.datetime.now() - self.progress_bar = None + self._progress_bar = None disable_progress_bar = self.cli_ctx.config.getboolean('core', 'disable_progress_bar', False) - if not disable_progress_bar and not cli_ctx.only_show_errors: - self.progress_bar = progress_bar if progress_bar is not None else IndeterminateProgressBar(cli_ctx) + self.disable_progress_bar = disable_progress_bar or cli_ctx.only_show_errors + if not self.disable_progress_bar: + self._progress_bar = progress_bar + + @property + def progress_bar(self): + if self.disable_progress_bar: + return None + if self._progress_bar is None: + from azure.cli.core.commands.progress import IndeterminateProgressBar + self._progress_bar = IndeterminateProgressBar(self.cli_ctx) + return self._progress_bar def _delay(self): time.sleep(self.poller_done_interval_ms / 1000.0) diff --git a/src/azure-cli-core/azure/cli/core/commands/parameters.py b/src/azure-cli-core/azure/cli/core/commands/parameters.py index c098d1a42a1..ad09007fbb3 100644 --- a/src/azure-cli-core/azure/cli/core/commands/parameters.py +++ b/src/azure-cli-core/azure/cli/core/commands/parameters.py @@ -5,7 +5,7 @@ import argparse -import platform +import sys from azure.cli.core import EXCLUDED_PARAMS from azure.cli.core.commands.constants import CLI_PARAM_KWARGS, CLI_POSITIONAL_PARAM_KWARGS @@ -274,7 +274,7 @@ def get_location_type(cli_ctx): validator=generate_deployment_name ) -quotes = '""' if platform.system() == 'Windows' else "''" +quotes = '""' if sys.platform == 'win32' else "''" quote_text = 'Use {} to clear existing tags.'.format(quotes) tags_type = CLIArgumentType( diff --git a/src/azure-cli-core/azure/cli/core/commands/progress.py b/src/azure-cli-core/azure/cli/core/commands/progress.py index 4a68389c619..f6ec6abfa5e 100644 --- a/src/azure-cli-core/azure/cli/core/commands/progress.py +++ b/src/azure-cli-core/azure/cli/core/commands/progress.py @@ -4,8 +4,6 @@ # -------------------------------------------------------------------------------------------- import sys -from humanfriendly.terminal.spinners import Spinner - BAR_LEN = 70 EMPTY_LINE = ' ' * BAR_LEN @@ -114,6 +112,7 @@ def write(self, args): :param args: dictionary containing key 'message' """ if self.spinner is None: + from humanfriendly.terminal.spinners import Spinner self.spinner = Spinner( # pylint: disable=no-member label='In Progress', stream=self.out, hide_cursor=False) msg = args.get('message', 'In Progress') @@ -178,10 +177,15 @@ def __init__(self, cli_ctx, message="Running"): self.message = message self.hook = self.cli_ctx.get_progress_controller( det=False, - spinner=Spinner( # pylint: disable=no-member - label='Running', - stream=sys.stderr, - hide_cursor=False)) + spinner=self._create_spinner()) + + @staticmethod + def _create_spinner(): + from humanfriendly.terminal.spinners import Spinner + return Spinner( # pylint: disable=no-member + label='Running', + stream=sys.stderr, + hide_cursor=False) def begin(self): self.hook.begin() diff --git a/src/azure-cli-core/azure/cli/core/commands/validators.py b/src/azure-cli-core/azure/cli/core/commands/validators.py index 466fb284bfe..43a02a1f13d 100644 --- a/src/azure-cli-core/azure/cli/core/commands/validators.py +++ b/src/azure-cli-core/azure/cli/core/commands/validators.py @@ -5,7 +5,6 @@ import argparse import time -import random from azure.cli.core.profiles import ResourceType @@ -67,6 +66,7 @@ def validate_key_value_pairs(string): def generate_deployment_name(namespace): if not namespace.deployment_name: + import random namespace.deployment_name = \ 'azurecli{}{}'.format(str(time.time()), str(random.randint(1, 100000))) diff --git a/src/azure-cli-core/azure/cli/core/decorators.py b/src/azure-cli-core/azure/cli/core/decorators.py index d59d0316912..71d7c37a0b1 100644 --- a/src/azure-cli-core/azure/cli/core/decorators.py +++ b/src/azure-cli-core/azure/cli/core/decorators.py @@ -10,7 +10,6 @@ that it doesn't import modules other than those in the Python Standard Library """ -import hashlib from functools import wraps from knack.log import get_logger @@ -43,6 +42,7 @@ def call_once(factory_func): def _wrapped(*args, **kwargs): if not factory_func.executed: factory_func.cached_result = factory_func(*args, **kwargs) + factory_func.executed = True return factory_func.cached_result @@ -57,6 +57,7 @@ def hash256_result(func): @wraps(func) def _decorator(*args, **kwargs): + import hashlib val = func(*args, **kwargs) if val is None: raise ValueError('Return value is None') diff --git a/src/azure-cli-core/azure/cli/core/extension/__init__.py b/src/azure-cli-core/azure/cli/core/extension/__init__.py index 627f3fdd5c7..53afb00ecd9 100644 --- a/src/azure-cli-core/azure/cli/core/extension/__init__.py +++ b/src/azure-cli-core/azure/cli/core/extension/__init__.py @@ -10,7 +10,6 @@ import re from sysconfig import get_path -import pkginfo from knack.config import CLIConfig from knack.log import get_logger from azure.cli.core._config import GLOBAL_CONFIG_DIR, ENV_VAR_PREFIX @@ -134,6 +133,7 @@ def get_version(self): def get_metadata(self): from glob import glob + import pkginfo metadata = {} ext_dir = self.path or get_extension_path(self.name) @@ -213,6 +213,8 @@ def get_version(self): return self.metadata.get('version') def get_metadata(self): + import pkginfo + metadata = {} ext_dir = self.path if not ext_dir or not os.path.isdir(ext_dir): diff --git a/src/azure-cli-core/azure/cli/core/extension/operations.py b/src/azure-cli-core/azure/cli/core/extension/operations.py index 83576a83d5f..e20aed58b03 100644 --- a/src/azure-cli-core/azure/cli/core/extension/operations.py +++ b/src/azure-cli-core/azure/cli/core/extension/operations.py @@ -13,7 +13,6 @@ import traceback import hashlib from subprocess import check_output, STDOUT, CalledProcessError -from urllib.parse import urlparse from packaging.version import parse @@ -85,6 +84,7 @@ def _validate_whl_extension(ext_file): def _get_extension_info_from_source(source): + from urllib.parse import urlparse url_parse_result = urlparse(source) is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https') whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source) @@ -96,6 +96,7 @@ def _get_extension_info_from_source(source): def _add_whl_ext(cli_ctx, source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None, system=None): # pylint: disable=too-many-statements + from urllib.parse import urlparse cli_ctx.get_progress_controller().add(message='Analyzing') if not source.endswith('.whl'): raise ValueError('Unknown extension type. Only Python wheels are supported.') diff --git a/src/azure-cli-core/azure/cli/core/local_context.py b/src/azure-cli-core/azure/cli/core/local_context.py index 1983c5ddc31..e2fa9f5d0e7 100644 --- a/src/azure-cli-core/azure/cli/core/local_context.py +++ b/src/azure-cli-core/azure/cli/core/local_context.py @@ -4,7 +4,6 @@ # -------------------------------------------------------------------------------------------- import os -import shutil import configparser import enum @@ -132,6 +131,7 @@ def delete_file(self, recursive=False): os.remove(local_context_file.config_path) parent_dir = os.path.dirname(local_context_file.config_path) if not os.listdir(parent_dir): + import shutil shutil.rmtree(parent_dir) logger.warning('Parameter persistence file in working directory %s is deleted.', os.path.dirname(local_context_file.config_dir)) diff --git a/src/azure-cli-core/azure/cli/core/telemetry.py b/src/azure-cli-core/azure/cli/core/telemetry.py index 21d79ecef36..aeb06ba35f2 100644 --- a/src/azure-cli-core/azure/cli/core/telemetry.py +++ b/src/azure-cli-core/azure/cli/core/telemetry.py @@ -131,6 +131,8 @@ def generate_payload(self): return _remove_symbols(payload) def _get_base_properties(self): + uname = platform.uname() + return { 'Reserved.ChannelUsed': 'AI', 'Reserved.EventId': self.event_id, @@ -151,9 +153,10 @@ def _get_base_properties(self): self.product_version, self.module_version), 'Context.Default.VS.Core.MacAddressHash': _get_hash_mac_address(), 'Context.Default.VS.Core.Machine.Id': _get_hash_machine_id(), - 'Context.Default.VS.Core.OS.Type': platform.system().lower(), # eg. darwin, windows - 'Context.Default.VS.Core.OS.Version': platform.version().lower(), # eg. 10.0.14942 - 'Context.Default.VS.Core.OS.Platform': platform.platform().lower(), # eg. windows-10-10.0.19041-sp0 + 'Context.Default.VS.Core.OS.Type': uname.system.lower(), # eg. darwin, windows + 'Context.Default.VS.Core.OS.Version': uname.version.lower(), # eg. 10.0.14942 + 'Context.Default.VS.Core.OS.Platform': '{}-{}-{}-{}'.format( + uname.system, uname.release, uname.version, uname.machine).lower(), # eg. windows-10-10.0.19041-sp0 # the distro info is complement of platform info for linux 'Context.Default.VS.Core.Distro.Name': _get_distro_name(), # eg. 'CentOS Linux 8' 'Context.Default.VS.Core.Distro.Id': _get_distro_id(), # eg. 'centos' diff --git a/src/azure-cli-core/azure/cli/core/util.py b/src/azure-cli-core/azure/cli/core/util.py index ce64858bd09..b693ee00518 100644 --- a/src/azure-cli-core/azure/cli/core/util.py +++ b/src/azure-cli-core/azure/cli/core/util.py @@ -4,18 +4,12 @@ # -------------------------------------------------------------------------------------------- # pylint: disable=too-many-lines -import base64 -import binascii -import getpass import json -import yaml import logging import os import platform import re -import ssl import sys -from urllib.request import urlopen from knack.log import get_logger from knack.util import CLIError, to_snake_case, to_camel_case @@ -569,6 +563,7 @@ def get_file_json(file_path, throw_on_empty=True, preserve_order=False): def get_file_yaml(file_path, throw_on_empty=True): + import yaml # Lazy-load: only needed when parsing YAML files content = read_file_content(file_path) if not content: if throw_on_empty: @@ -593,6 +588,7 @@ def read_file_content(file_path, allow_binary=False): if allow_binary: try: + import base64 with open(file_path, 'rb') as input_file: logger.debug("attempting to read file %s as binary", file_path) return base64.b64encode(input_file.read()).decode("utf-8") @@ -643,6 +639,7 @@ def b64encode(s): :return: base64 encoded string :rtype: str """ + import base64 encoded = base64.b64encode(s.encode("latin-1")) return encoded.decode('latin-1') @@ -654,6 +651,7 @@ def b64decode(s): :return: decoded string :rtype: str """ + import base64 encoded = base64.b64decode(s.encode("latin-1")) return encoded.decode('latin-1') @@ -665,6 +663,8 @@ def b64_to_hex(s): :return: uppercase hex string :rtype: str """ + import base64 + import binascii decoded = base64.b64decode(s) hex_data = binascii.hexlify(decoded).upper() if isinstance(hex_data, bytes): @@ -883,6 +883,7 @@ def reload_module(module): def get_default_admin_username(): try: + import getpass username = getpass.getuser() except KeyError: username = None @@ -1200,10 +1201,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _ssl_context(): + import ssl return ssl.create_default_context() def urlretrieve(url): + from urllib.request import urlopen req = urlopen(url, context=_ssl_context()) return req.read() @@ -1332,17 +1335,19 @@ def handle_version_update(): """ try: from azure.cli.core._session import VERSIONS - from packaging.version import parse # pylint: disable=import-error,no-name-in-module from azure.cli.core import __version__ if not VERSIONS['versions']: get_cached_latest_versions() - elif parse(VERSIONS['versions']['core']['local']) != parse(__version__): - logger.debug("Azure CLI has been updated.") - logger.debug("Clean up versions and refresh cloud endpoints information in local files.") - VERSIONS['versions'] = {} - VERSIONS['update_time'] = '' - from azure.cli.core.cloud import refresh_known_clouds - refresh_known_clouds() + elif VERSIONS['versions']['core']['local'] != __version__: + # Lazy import packaging.version + from packaging.version import parse # pylint: disable=import-error,no-name-in-module + if parse(VERSIONS['versions']['core']['local']) != parse(__version__): + logger.debug("Azure CLI has been updated.") + logger.debug("Clean up versions and refresh cloud endpoints information in local files.") + VERSIONS['versions'] = {} + VERSIONS['update_time'] = '' + from azure.cli.core.cloud import refresh_known_clouds + refresh_known_clouds() except Exception as ex: # pylint: disable=broad-except logger.warning(ex) diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/__init__.py b/src/azure-cli-telemetry/azure/cli/telemetry/__init__.py index b9eb7ec1f86..fcc931d536f 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/__init__.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/__init__.py @@ -5,8 +5,6 @@ import sys import os -import subprocess -import portalocker from azure.cli.telemetry.util import save_payload @@ -16,6 +14,7 @@ def _start(config_dir, cache_dir): + import subprocess from azure.cli.telemetry.components.telemetry_logging import get_logger logger = get_logger('process') @@ -100,6 +99,7 @@ def main(): logger.info('Attempt start. Configuration directory [%s]. Cache directory [%s].', sys.argv[1], sys.argv[2]) try: + import portalocker collection = RecordsCollection(cache_dir) collection.snapshot_and_read() diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/components/telemetry_client.py b/src/azure-cli-telemetry/azure/cli/telemetry/components/telemetry_client.py index 5caeed563dc..b654da5526d 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/components/telemetry_client.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/components/telemetry_client.py @@ -6,9 +6,6 @@ import json import datetime -import urllib.request as http_client_t -from urllib.error import HTTPError - from applicationinsights import TelemetryClient from applicationinsights.channel import SynchronousSender, SynchronousQueue, TelemetryChannel @@ -92,6 +89,9 @@ def __init__(self): def send(self, data_to_send): """ Override the default resend mechanism in SenderBase. Stop resend when it fails.""" + import urllib.request as http_client_t + from urllib.error import HTTPError + request_payload = json.dumps([a.write() for a in data_to_send]) content = bytearray(request_payload, 'utf-8') diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/util.py b/src/azure-cli-telemetry/azure/cli/telemetry/util.py index 8c0a44a81e1..d9e72ed3f04 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/util.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/util.py @@ -5,7 +5,6 @@ import os import logging -import logging.handlers def save_payload(config_dir, payload): @@ -25,6 +24,7 @@ def save_payload(config_dir, payload): def _create_rotate_file_logger(log_dir): + from logging.handlers import RotatingFileHandler from datetime import datetime now = datetime.now() cache_name = os.path.join(log_dir, 'telemetry', now.strftime('%Y%m%d%H%M%S%f')[:-3], 'cache') @@ -32,7 +32,7 @@ def _create_rotate_file_logger(log_dir): if not os.path.exists(os.path.dirname(cache_name)): os.makedirs(os.path.dirname(cache_name)) - handler = logging.handlers.RotatingFileHandler(cache_name, maxBytes=128 * 1024, backupCount=100) + handler = RotatingFileHandler(cache_name, maxBytes=128 * 1024, backupCount=100) handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter(fmt='%(asctime)s,%(message)s', datefmt='%Y-%m-%dT%H:%M:%S'))