[NetAppFiles] support 0.5 tib pool

azure-pipelines.yml

@@ -1159,6 +1159,24 @@ jobs:
   #     azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account"
   #   displayName: "Execution Performance"
 
+- job: CheckExternalUrls
+  displayName: "Check External Source URLs"
+  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+  pool:
+    name: ${{ variables.ubuntu_pool }}
+  steps:
+    - task: UsePythonVersion@0
+      displayName: 'Use Python 3.13'
+      inputs:
+        versionSpec: 3.13
+    - bash: |
+        #!/usr/bin/env bash
+        set -ev
+        # External URL exclusions are maintained in scripts/ci/external_url_exclusions.json.
+        git fetch origin --depth=1 $(System.PullRequest.TargetBranch)
+        python scripts/ci/validate_external_source_urls.py --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch)
+      displayName: 'Validate External Source URLs'
+
 - job: CheckLinter
   displayName: "Check CLI Linter"
 

scripts/ci/external_url_exclusions.json

@@ -0,0 +1,11 @@
+{
+  "tool": "External URL Validation",
+  "scope": {
+    "include": ["src/**/*.py"],
+    "exclude": [
+      "**/tests/**",
+      "**/vendored_sdks/**",
+      "**/*help.py"
+    ]
+  }
+}

scripts/ci/validate_external_source_urls.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+"""Fail CI if forbidden raw GitHub URL is introduced in new diff lines."""
+
+import argparse
+import fnmatch
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+GITHUB_URL_PATTERN = re.compile(
+    r"https?://raw\.githubusercontent\.com/[^\s\"'`,)}\]]*"
+)
+INLINE_SUPPRESSION_PATTERN = re.compile(
+    r"#\s*external-url-exempt:\s*\S"
+)
+_FILENAME_PATTERN = re.compile(r"^[A-Za-z0-9_\-]+\.[A-Za-z0-9]{1,10}$")
+RECOMMENDED_INTERNAL_URL = "https://azcliprod.blob.core.windows.net/cli"
+SCOPE_CONFIG_PATH = Path(__file__).with_name("external_url_exclusions.json")
+
+# Scope configuration loaded from external_url_exclusions.json.
+# Contains optional "include" and "exclude" glob-pattern lists.
+_SCOPE_CONFIG = None
+
+
+def _load_scope_config():
+    """Load scope configuration (include/exclude patterns) from the JSON file."""
+    try:
+        with SCOPE_CONFIG_PATH.open(encoding="utf-8") as input_file:
+            config = json.load(input_file)
+    except (OSError, ValueError) as ex:
+        raise RuntimeError(f"Unable to load scope config from '{SCOPE_CONFIG_PATH}': {ex}") from ex
+
+    if not isinstance(config, dict):
+        raise RuntimeError(
+            f"Invalid scope configuration in '{SCOPE_CONFIG_PATH}': expected a JSON object"
+        )
+
+    scope = config.get("scope", {})
+    if not isinstance(scope, dict):
+        raise RuntimeError(
+            f"Invalid scope configuration in '{SCOPE_CONFIG_PATH}': 'scope' must be a JSON object"
+        )
+
+    include = scope.get("include", [])
+    exclude = scope.get("exclude", [])
+
+    if isinstance(include, str):
+        include = [include]
+    if isinstance(exclude, str):
+        exclude = [exclude]
+
+    if not isinstance(include, list) or not all(isinstance(p, str) for p in include):
+        raise RuntimeError(
+            f"Invalid scope configuration in '{SCOPE_CONFIG_PATH}': 'include' must be a string or array of strings"
+        )
+    if not isinstance(exclude, list) or not all(isinstance(p, str) for p in exclude):
+        raise RuntimeError(
+            f"Invalid scope configuration in '{SCOPE_CONFIG_PATH}': 'exclude' must be a string or array of strings"
+        )
+
+    return (
+        [p.replace("\\", "/") for p in include],
+        [p.replace("\\", "/") for p in exclude],
+    )
+
+
+def _get_scope_config():
+    """Return cached (include_patterns, exclude_patterns) tuple."""
+    global _SCOPE_CONFIG  # pylint: disable=global-statement
+
+    if _SCOPE_CONFIG is None:
+        _SCOPE_CONFIG = _load_scope_config()
+
+    return _SCOPE_CONFIG
+
+
+def _matches_any(file_path: str, patterns: list) -> bool:
+    """Return True if *file_path* matches any of the given glob patterns."""
+    return any(fnmatch.fnmatch(file_path, p) for p in patterns)
+
+
+
+def _extract_filename_from_url(line: str) -> str:
+    """Extract the file name from the first GitHub URL found in *line*.
+
+    Returns the basename (e.g. ``map.json``) or ``"xxx.xxx"`` when no
+    recognisable file name is present.
+    """
+    match = GITHUB_URL_PATTERN.search(line)
+    if match:
+        url_path = match.group(0).rstrip("/")
+        basename = url_path.rsplit("/", 1)[-1] if "/" in url_path else ""
+        if _FILENAME_PATTERN.match(basename):
+            return basename
+    return "xxx.xxx"
+
+
+def _should_flag(file_path: str) -> bool:
+    """Decide whether *file_path* should be checked for forbidden URLs.
+
+    An entry is included when there is no include list (empty means
+    "entire codebase") or when it matches at least one include pattern.
+    A included entry is then flagged unless it also matches an exclude pattern.
+    """
+    include_patterns, exclude_patterns = _get_scope_config()
+
+    included = (not include_patterns) or _matches_any(file_path, include_patterns)
+    return included and not _matches_any(file_path, exclude_patterns)
+
+
+def _run_diff(src: str, tgt: str, cached: bool = False) -> str:
+    cmd = ["git", "diff", "--unified=0", "--no-color"]
+    if cached:
+        cmd.append("--cached")
+    else:
+        cmd.append(f"{tgt}...{src}")
+
+    proc = subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        check=False,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError(proc.stderr.strip() or "git diff failed")
+    return proc.stdout
+
+
+def _find_violations(diff_text: str):
+    violations = []
+    current_file = ""
+    prev_added_line = ""
+
+    for line in diff_text.splitlines():
+        if line.startswith("+++ b/"):
+            current_file = line[6:]
+            prev_added_line = ""
+            continue
+
+        if not line.startswith("+") or line.startswith("+++"):
+            prev_added_line = ""
+            continue
+
+        added_line = line[1:]
+        if GITHUB_URL_PATTERN.search(added_line) and _should_flag(current_file):
+            # Skip if the current line or the previous added line has a suppression comment
+            if not (INLINE_SUPPRESSION_PATTERN.search(added_line)
+                    or INLINE_SUPPRESSION_PATTERN.search(prev_added_line)):
+                violations.append((current_file or "<unknown>", added_line.strip()))
+
+        prev_added_line = added_line
+
+    return violations
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Check diff for forbidden raw GitHub URL usage.")
+    parser.add_argument("--src", default="HEAD", help="Source ref/commit for git diff.")
+    parser.add_argument("--tgt", default="HEAD~1", help="Target ref/commit for git diff.")
+    parser.add_argument("--cached", action="store_true", help="Check staged changes in git index.")
+    args = parser.parse_args()
+
+    try:
+        _get_scope_config()
+        diff_text = _run_diff(src=args.src, tgt=args.tgt, cached=args.cached)
+    except Exception as ex:  # pylint: disable=broad-except
+        if args.cached:
+            print(f"Unable to evaluate staged diff: {ex}", file=sys.stderr)
+        else:
+            print(f"Unable to evaluate diff between '{args.tgt}' and '{args.src}': {ex}", file=sys.stderr)
+        return 1
+
+    violations = _find_violations(diff_text)
+    if not violations:
+        print("No forbidden external GitHub URL found in added lines.")
+        return 0
+
+    print("ERROR: Found forbidden external GitHub URL(s) in this change:\n", file=sys.stderr)
+    for file_path, content in violations:
+        filename = _extract_filename_from_url(content)
+        print(
+            f"  {file_path}: {content}\n"
+            "\n"
+            "  To fix, follow one of the options below (in priority order):\n"
+            "\n"
+            "    Option 1 (Preferred) — Host the file in the AME storage account\n"
+            "    ---------------------------------------------------------------\n"
+            "    Reach out to the Platform squad to upload the file to the shared\n"
+            "    Azure CLI storage account. Once uploaded, replace the raw GitHub\n"
+            "    URL with the internal blob URL. The resulting URL should look like:\n"
+            "\n"
+            f"      {RECOMMENDED_INTERNAL_URL}/<module>/{filename}\n"
+            "\n"
+            "    Option 2 (Fallback) — Suppress with an inline comment\n"
+            "    -----------------------------------------------------\n"
+            "    Only if the GitHub URL is required by design (e.g. the upstream\n"
+            "    repo IS the authoritative source), add an inline suppression\n"
+            "    comment on the line before or on the same line like:\n"
+            "\n"
+            "      # external-url-exempt: <reason>\n"
+            f"     {content} \n",
+            file=sys.stderr,
+        )
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

src/azure-cli-core/azure/cli/core/azlogging.py

@@ -26,6 +26,7 @@
 import os
 import logging
 import datetime
+from logging.handlers import RotatingFileHandler
 
 from azure.cli.core.commands.events import EVENT_INVOKER_PRE_CMD_TBL_TRUNCATE
 
@@ -38,6 +39,20 @@
 _CMD_LOG_LINE_PREFIX = "CMD-LOG-LINE-BEGIN"
 
 
+class SecureFileHandler(logging.FileHandler):
+    """A FileHandler that creates the log file with 600 permissions (owner read/write only)."""
+    def _open(self):
+        fd = os.open(self.baseFilename, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+        return os.fdopen(fd, self.mode, encoding=self.encoding)
+
+
+class SecureRotatingFileHandler(RotatingFileHandler):
+    """A RotatingFileHandler that creates log files with 600 permissions (owner read/write only)."""
+    def _open(self):
+        fd = os.open(self.baseFilename, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+        return os.fdopen(fd, self.mode, encoding=self.encoding)
+
+
 class AzCliLogging(CLILogging):
     COMMAND_METADATA_LOGGER = 'az_command_data_logger'
 
@@ -58,6 +73,20 @@ def configure(self, args):
             # when debug log is shown.
             logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(logging.CRITICAL)
 
+    def _init_logfile_handlers(self, root_logger, cli_loggers):
+        # Override knack's CLILogging._init_logfile_handlers() (called by CLILogging.configure())
+        # to use SecureRotatingFileHandler, ensuring log files are created with 0o600 permissions.
+        ensure_dir(self.log_dir)
+        log_file_path = os.path.join(self.log_dir, self.logfile_name)
+        logfile_handler = SecureRotatingFileHandler(log_file_path, maxBytes=10 * 1024 * 1024, backupCount=5,
+                                                    encoding=LOG_FILE_ENCODING)
+        lfmt = logging.Formatter('%(process)d : %(asctime)s : %(levelname)s : %(name)s : %(message)s')
+        logfile_handler.setFormatter(lfmt)
+        logfile_handler.setLevel(logging.DEBUG)
+        root_logger.addHandler(logfile_handler)
+        for cli_logger in cli_loggers:
+            cli_logger.addHandler(logfile_handler)
+
     def get_command_log_dir(self):
         return self.command_log_dir
 
@@ -112,7 +141,7 @@ def _init_command_logfile_handlers(self, command_metadata_logger, args):
         log_file_path = os.path.join(self.command_log_dir, log_name)
         get_logger(__name__).debug("metadata file logging enabled - writing logs to '%s'.", log_file_path)
 
-        logfile_handler = logging.FileHandler(log_file_path, encoding=LOG_FILE_ENCODING)
+        logfile_handler = SecureFileHandler(log_file_path, encoding=LOG_FILE_ENCODING)
 
         lfmt = logging.Formatter(_CMD_LOG_LINE_PREFIX + ' %(process)d | %(asctime)s | %(levelname)s | %(name)s | %(message)s')  # pylint: disable=line-too-long
         logfile_handler.setFormatter(lfmt)

src/azure-cli-core/azure/cli/core/cloud.py

@@ -502,7 +502,7 @@ class CloudNameEnum:  # pylint: disable=too-few-public-methods
         active_directory_resource_id='https://management.sovcloud-api.fr/',
         active_directory_graph_resource_id='https://graph.svc.sovcloud.fr/',
         microsoft_graph_resource_id='https://graph.svc.sovcloud.fr',
-        vm_image_alias_doc='https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json',
+        vm_image_alias_doc='https://azcliprod.blob.core.windows.net/cli/vm/aliases_master.json',
         media_resource_id='https://rest.media.sovcloud-api.fr',
         ossrdbms_resource_id='https://ossrdbms-aad.database.sovcloud-api.fr',
         portal='https://portal.sovcloud-azure.fr'),

src/azure-cli-core/azure/cli/core/commandIndex.latest.json

@@ -104,9 +104,6 @@
     "databoxedge": [
       "azure.cli.command_modules.databoxedge"
     ],
-    "demo": [
-      "azure.cli.command_modules.util"
-    ],
     "deployment": [
       "azure.cli.command_modules.resource"
     ],

src/azure-cli-core/azure/cli/core/style.py

@@ -13,8 +13,6 @@
 
 Console Virtual Terminal Sequences:
 https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#text-formatting
-
-For a complete demo, see `src/azure-cli/azure/cli/command_modules/util/custom.py` and run `az demo style`.
 """
 
 import sys