From fa00b079d1ed0b4acaaeb9e79550e646e837009d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 03:25:18 +0000 Subject: [PATCH 1/6] Initial plan From 4b7918261200ea4fad234b40fc75f278eaab8e88 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 03:40:34 +0000 Subject: [PATCH 2/6] Remove AutoRest generation path; retain only TypeSpec path in eng/automation Co-authored-by: XiaofeiCao <92354331+XiaofeiCao@users.noreply.github.com> --- eng/automation/generate.py | 222 ++------------- eng/automation/generate_data.py | 471 +------------------------------ eng/automation/generate_utils.py | 190 ------------- eng/automation/generation.yml | 32 +-- eng/automation/parameters.py | 49 ---- eng/automation/sdk_generate.py | 314 --------------------- eng/automation/sdk_init.sh | 8 - eng/codegen_to_sdk_config.json | 13 - 8 files changed, 20 insertions(+), 1279 deletions(-) delete mode 100755 eng/automation/sdk_generate.py delete mode 100755 eng/automation/sdk_init.sh delete mode 100644 eng/codegen_to_sdk_config.json diff --git a/eng/automation/generate.py b/eng/automation/generate.py index 79dc46065eb6..f5dd91b3747c 100755 --- a/eng/automation/generate.py +++ b/eng/automation/generate.py @@ -21,48 +21,19 @@ is_first_release, ) from generate_data import ( - sdk_automation as sdk_automation_data, sdk_automation_typespec_project as sdk_automation_typespec_project_data, ) from generate_utils import ( compare_with_maven_package, compile_arm_package, - generate, - get_and_update_service_from_api_specs, - get_suffix_from_api_specs, - update_spec, generate_typespec_project, - is_mgmt_premium, ) os.chdir(pwd) -def update_parameters(suffix): - # update changeable parameters in parameters.py - global SUFFIX, NAMESPACE_SUFFIX, ARTIFACT_SUFFIX, NAMESPACE_FORMAT, ARTIFACT_FORMAT, OUTPUT_FOLDER_FORMAT - - SUFFIX = suffix - - NAMESPACE_SUFFIX = ".{0}".format(SUFFIX) if SUFFIX else "" - ARTIFACT_SUFFIX = "-{0}".format(SUFFIX) if SUFFIX else "" - NAMESPACE_FORMAT = "com.azure.resourcemanager.{{0}}{0}".format(NAMESPACE_SUFFIX) - ARTIFACT_FORMAT = "azure-resourcemanager-{{0}}{0}".format(ARTIFACT_SUFFIX) - OUTPUT_FOLDER_FORMAT = "sdk/{{0}}/{0}".format(ARTIFACT_FORMAT) - - def parse_args() -> (argparse.ArgumentParser, argparse.Namespace): parser = argparse.ArgumentParser() - parser.add_argument( - "--spec-root", - default="https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/", - help="Spec root folder", - ) - parser.add_argument( - "-r", - "--readme", - help='Readme path, Sample: "storage" or "specification/storage/resource-manager/readme.md"', - ) parser.add_argument( "-c", "--tsp-config", @@ -70,30 +41,7 @@ def parse_args() -> (argparse.ArgumentParser, argparse.Namespace): "Currently only support remote url with specific commitID " "e.g. https://github.com/Azure/azure-rest-api-specs/blob/042e4045dedff4baaf5ae551bf6c8087fbdacd40/specification/deviceregistry/DeviceRegistry.Management/tspconfig.yaml", ) - parser.add_argument("-t", "--tag", help="Specific tag") parser.add_argument("-v", "--version", help="Specific sdk version") - parser.add_argument( - "-s", - "--service", - help="Service Name if not the same as spec name", - ) - parser.add_argument( - "-u", - "--use", - default=AUTOREST_JAVA, - help="Autorest java plugin", - ) - parser.add_argument( - "--autorest", - default=AUTOREST_CORE_VERSION, - help="Autorest version", - ) - parser.add_argument( - "--autorest-options", - default="", - help="Additional autorest options", - ) - parser.add_argument("--suffix", help="Suffix for namespace and artifact") parser.add_argument( "--auto-commit-external-change", action="store_true", @@ -119,9 +67,6 @@ def sdk_automation(input_file: str, output_file: str): try: # typespec packages = sdk_automation_typespec(config) - # autorest - if not packages: - packages = sdk_automation_autorest(config) except ValueError: logging.error("[VALIDATION] Parameter validation failed.", exc_info=True) sys.exit(1) @@ -141,109 +86,6 @@ def sdk_automation(input_file: str, output_file: str): sys.exit(1) -def sdk_automation_autorest(config: dict) -> List[dict]: - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - api_specs_file = os.path.join(base_dir, API_SPECS_FILE) - - packages = [] - breaking = False - changelog = "" - breaking_change_items = [] - if "relatedReadmeMdFiles" not in config or not config["relatedReadmeMdFiles"]: - return packages - - for readme in config["relatedReadmeMdFiles"]: - match = re.search( - r"specification/([^/]+)/resource-manager((?:/[^/]+)*)/readme.md", - readme, - re.IGNORECASE, - ) - if not match: - logging.info("[Skip] readme path does not format as specification/*/resource-manager/*/readme.md") - else: - spec = match.group(1) - spec = update_spec(spec, match.group(2)) - service = get_and_update_service_from_api_specs(api_specs_file, spec, truncate_service=True) - - pre_suffix = SUFFIX - suffix = get_suffix_from_api_specs(api_specs_file, spec) - if suffix is None: - suffix = SUFFIX - update_parameters(suffix) - - # TODO: use specific function to detect tag in "resources" spec/service - tag = None - if service == "resources" and spec == service: - with open(os.path.join(config["specFolder"], readme)) as fin: - tag_match = re.search(r"tag: (package-resources-\S+)", fin.read()) - if tag_match: - tag = tag_match.group(1) - else: - tag = "package-resources-2025-04" - - module = ARTIFACT_FORMAT.format(service) - output_folder = OUTPUT_FOLDER_FORMAT.format(service) - namespace = NAMESPACE_FORMAT.format(service) - stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module) - succeeded = generate( - sdk_root, - service, - spec_root=config["specFolder"], - readme=readme, - autorest=AUTOREST_CORE_VERSION, - use=AUTOREST_JAVA, - output_folder=output_folder, - module=module, - namespace=namespace, - tag=tag, - premium=is_mgmt_premium(module), - ) - if succeeded: - succeeded = compile_arm_package(sdk_root, module) - if succeeded: - stable_version = get_latest_ga_version(GROUP_ID, module, stable_version) - breaking, changelog, breaking_change_items = compare_with_maven_package( - sdk_root, GROUP_ID, service, stable_version, current_version, module - ) - - packages.append( - { - "packageName": "{0}".format(ARTIFACT_FORMAT.format(service)), - "path": [ - output_folder, - CI_FILE_FORMAT.format(service), - POM_FILE_FORMAT.format(service), - "eng/versioning", - "pom.xml", - ], - "readmeMd": [readme], - "artifacts": ( - ["{0}/pom.xml".format(output_folder)] - + [jar for jar in glob.glob("{0}/target/*.jar".format(output_folder))] - if succeeded - else [] - ), - "apiViewArtifact": next(iter(glob.glob("{0}/target/*-sources.jar".format(output_folder))), None), - "language": "Java", - "result": "succeeded" if succeeded else "failed", - "changelog": { - "content": changelog, - "hasBreakingChange": breaking, - "breakingChangeItems": breaking_change_items, - }, - } - ) - - update_parameters(pre_suffix) - - if not packages: - # try data-plane codegen - packages = sdk_automation_data(config) - - return packages - - def sdk_automation_typespec(config: dict) -> List[dict]: packages = [] @@ -496,59 +338,27 @@ def main(): base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - api_specs_file = os.path.join(base_dir, API_SPECS_FILE) - if args.get("tsp_config"): - tsp_config = args["tsp_config"] + if not args.get("tsp_config"): + parser.print_help() + sys.exit(0) - succeeded, require_sdk_integration, sdk_folder, service, module = generate_typespec_project( - tsp_project=tsp_config, sdk_root=sdk_root, remove_before_regen=True, group_id=GROUP_ID, **args - ) + tsp_config = args["tsp_config"] - stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module, **args) - args["version"] = current_version + succeeded, require_sdk_integration, sdk_folder, service, module = generate_typespec_project( + tsp_project=tsp_config, sdk_root=sdk_root, remove_before_regen=True, group_id=GROUP_ID, **args + ) - if require_sdk_integration: - update_service_files_for_new_lib(sdk_root, service, GROUP_ID, module) - update_root_pom(sdk_root, service) + stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module, **args) + args["version"] = current_version - output_folder = sdk_folder - update_version(sdk_root, output_folder) - update_changelog_version(sdk_root, output_folder, current_version) - else: - if not args.get("readme"): - parser.print_help() - sys.exit(0) - - readme = args["readme"] - match = re.match( - r"specification/([^/]+)/resource-manager((?:/[^/]+)*)/readme.md", - readme, - re.IGNORECASE, - ) - if not match: - spec = readme - readme = "specification/{0}/resource-manager/readme.md".format(spec) - else: - spec = match.group(1) - spec = update_spec(spec, match.group(2)) - - args["readme"] = readme - args["spec"] = spec - - suffix = args.get("suffix") or get_suffix_from_api_specs(api_specs_file, spec) - update_parameters(suffix) - service = get_and_update_service_from_api_specs(api_specs_file, spec, args["service"], suffix) - args["service"] = service - module = ARTIFACT_FORMAT.format(service) - premium = is_mgmt_premium(module) - stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module, **args) - args["version"] = current_version - output_folder = OUTPUT_FOLDER_FORMAT.format(service) - namespace = NAMESPACE_FORMAT.format(service) - succeeded = generate( - sdk_root, module=module, output_folder=output_folder, namespace=namespace, premium=premium, **args - ) + if require_sdk_integration: + update_service_files_for_new_lib(sdk_root, service, GROUP_ID, module) + update_root_pom(sdk_root, service) + + output_folder = sdk_folder + update_version(sdk_root, output_folder) + update_changelog_version(sdk_root, output_folder, current_version) if succeeded: succeeded = compile_arm_package(sdk_root, module) diff --git a/eng/automation/generate_data.py b/eng/automation/generate_data.py index 90a00e92e0be..182ae8063ef4 100755 --- a/eng/automation/generate_data.py +++ b/eng/automation/generate_data.py @@ -1,29 +1,20 @@ #!/usr/bin/env python3 import os import sys -import shutil -import logging -import argparse -import re import glob -import subprocess -import yaml -import requests +import logging + from utils import get_latest_ga_version from generate_utils import compare_with_maven_package -from typing import List, Tuple, Optional from parameters import * from utils import set_or_default_version from utils import update_service_files_for_new_lib from utils import update_root_pom -from utils import ListIndentDumper from generate_utils import generate_typespec_project, clean_sdk_folder GROUP_ID = "com.azure" -DPG_ARGUMENTS = "--sdk-integration --generate-samples --generate-tests" -YAML_BLOCK_REGEX = r"```\s?(?:yaml|YAML).*?\n(.*?)```" def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict: @@ -137,277 +128,6 @@ def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict: } -def get_or_update_sdk_readme(config: dict, readme_file_path: str) -> Optional[str]: - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - sdk_readme_abspath = None - - if "autorestConfig" in config: - # autorestConfig - - autorest_config: str = config["autorestConfig"] - - # find 'output-folder', and write swagger/README.md - autorest_config = autorest_config.replace(r"\r\n", r"\n") - yaml_blocks = re.findall(YAML_BLOCK_REGEX, autorest_config, re.DOTALL) - for yaml_str in yaml_blocks: - yaml_json = yaml.safe_load(yaml_str) - if "output-folder" in yaml_json: - output_folder: str = yaml_json["output-folder"] - if output_folder.startswith("sdk/"): - sdk_readme_abspath = os.path.join(sdk_root, output_folder, "swagger", "README.md") - os.makedirs(os.path.dirname(sdk_readme_abspath), exist_ok=True) - with open(sdk_readme_abspath, "w", encoding="utf-8") as f_out: - f_out.write(autorest_config) - logging.info("[RESOLVE] Create README from autorestConfig") - break - - if not sdk_readme_abspath: - # swagger/README.md in sdk repository - - # find all swagger/README.md in sdk repo - candidate_sdk_readmes = glob.glob(os.path.join(sdk_root, "sdk/*/*/swagger/README.md")) - # find the README.md that matches - sdk_readme_abspath = find_sdk_readme(readme_file_path, candidate_sdk_readmes) - - return sdk_readme_abspath - - -def sdk_automation(config: dict) -> List[dict]: - # priority: - # 1. autorestConfig from input - # 2. swagger/README.md in sdk repository that matches readme from input - - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - spec_root = os.path.abspath(config["specFolder"]) - - packages = [] - - # find readme.md in spec repository - readme_file_paths = [] - for file_path in config["relatedReadmeMdFiles"]: - match = re.search( - r"specification/([^/]+)/data-plane((?:/[^/]+)*)/readme.md", - file_path, - re.IGNORECASE, - ) - if match: - readme_file_paths.append(file_path) - - # readme.md required - if not readme_file_paths: - return packages - # we only take first readme.md - readme_file_path = readme_file_paths[0] - logging.info("[RESOLVE] README from specification %s", readme_file_path) - - sdk_readme_abspath = get_or_update_sdk_readme(config, readme_file_path) - - if sdk_readme_abspath: - spec_readme_abspath = os.path.join(spec_root, readme_file_path) - update_readme(sdk_readme_abspath, spec_readme_abspath) - sdk_automation_readme(sdk_readme_abspath, packages, sdk_root) - - return packages - - -def find_sdk_readme(spec_readme: str, candidate_sdk_readmes: List[str]) -> Optional[str]: - segments = spec_readme.split("/") - if "data-plane" in segments: - index = segments.index("data-plane") - # include service name, exclude readme.md - search_target = "/" + "/".join(segments[index - 1 :]) - - for sdk_readme_path in candidate_sdk_readmes: - spec_reference = find_sdk_spec_reference(sdk_readme_path) - if spec_reference and search_target in spec_reference: - return sdk_readme_path - return None - - -def find_sdk_spec_reference(sdk_readme_path: str) -> Optional[str]: - with open(sdk_readme_path, "r", encoding="utf-8") as f_in: - content = f_in.read() - if content: - yaml_blocks = re.findall(YAML_BLOCK_REGEX, content, re.DOTALL) - for yaml_str in yaml_blocks: - try: - yaml_json = yaml.safe_load(yaml_str) - if "data-plane" in yaml_json and yaml_json["data-plane"]: - # take 'require' - if "require" in yaml_json: - require = yaml_json["require"] - if isinstance(require, List): - require = require[0] - return require - # take 'input-file', if 'require' not found - if "input-file" in yaml_json: - input_file = yaml_json["input-file"] - if isinstance(input_file, List): - input_file = input_file[0] - return input_file - except yaml.YAMLError: - continue - return None - - -def sdk_automation_readme(readme_file_abspath: str, packages: List[dict], sdk_root: str): - service, module = get_generate_parameters(readme_file_abspath) - - if module: - succeeded = generate( - sdk_root, service, module, readme=readme_file_abspath, autorest=AUTOREST_CORE_VERSION, use=AUTOREST_JAVA - ) - - generated_folder = "sdk/{0}/{1}".format(service, module) - - breaking = False - changelog = "" - breaking_change_items = [] - - if succeeded: - stable_version, current_version = set_or_default_version(sdk_root, GROUP_ID, module) - succeeded = compile_package(sdk_root, GROUP_ID, module) - if succeeded: - breaking, changelog, breaking_change_items = compare_with_maven_package( - sdk_root, - GROUP_ID, - service, - get_latest_ga_version(GROUP_ID, module, stable_version), - current_version, - module, - ) - - artifacts = ["{0}/pom.xml".format(generated_folder)] - artifacts += [jar for jar in glob.glob("{0}/target/*.jar".format(generated_folder))] - result = "succeeded" if succeeded else "failed" - - packages.append( - { - "packageName": module, - "path": [ - generated_folder, - CI_FILE_FORMAT.format(service), - POM_FILE_FORMAT.format(service), - "eng/versioning", - "pom.xml", - ], - "artifacts": artifacts if succeeded else [], - "apiViewArtifact": next(iter(glob.glob("{0}/target/*-sources.jar".format(generated_folder))), None), - "language": "Java", - "result": result, - "changelog": { - "content": changelog, - "hasBreakingChange": breaking, - "breakingChangeItems": breaking_change_items, - }, - } - ) - - -def generate( - sdk_root: str, - service: str, - module: str, - *, - input_file: str = None, - spec_readme: str = None, - security: str = None, - security_scopes: str = None, - title: str = None, - autorest: str, - use: str, - autorest_options: str = "", - readme: str = None, -) -> bool: - namespace = "com.{0}".format(module.replace("-", ".")) - output_dir = os.path.join(sdk_root, "sdk", service, module) - # shutil.rmtree(os.path.join(output_dir, 'src/main'), ignore_errors=True) - shutil.rmtree( - os.path.join(output_dir, "src/samples/java", namespace.replace(".", "/"), "generated"), ignore_errors=True - ) - shutil.rmtree( - os.path.join(output_dir, "src/test/java", namespace.replace(".", "/"), "generated"), ignore_errors=True - ) - - if readme: - # use readme from spec repo - readme_file_path = readme - - require_sdk_integration = not os.path.exists(os.path.join(output_dir, "src")) - - logging.info("[GENERATE] Autorest from README {}".format(readme_file_path)) - - command = ( - "autorest --version={0} --use={1} --java --data-plane " - "--java.java-sdks-folder={2} --java.output-folder={3} {4} {5}".format( - autorest, - use, - os.path.abspath(sdk_root), - os.path.abspath(output_dir), - readme_file_path, - autorest_options, - ) - ) - if require_sdk_integration: - command += " --java.namespace={0} ".format(namespace) + DPG_ARGUMENTS - logging.info(command) - try: - subprocess.run(command, shell=True, check=True) - except subprocess.CalledProcessError: - logging.error("[GENERATE] Code generation failed. Autorest fails.") - return False - - if require_sdk_integration: - set_or_default_version(sdk_root, GROUP_ID, module) - update_service_files_for_new_lib(sdk_root, service, GROUP_ID, module) - update_root_pom(sdk_root, service) - else: - # no readme - - security_arguments = "" - if security: - security_arguments += "--security={0}".format(security) - if security_scopes: - security_arguments += " --security-scopes={0}".format(security_scopes) - - if spec_readme: - logging.info("[GENERATE] Autorest from README {}".format(spec_readme)) - input_arguments = "--require={0}".format(spec_readme) - else: - logging.info("[GENERATE] Autorest from JSON {}".format(input_file)) - input_arguments = "--input-file={0}".format(input_file) - - artifact_arguments = "--artifact-id={0}".format(module) - if title: - artifact_arguments += " --title={0}".format(title) - - command = ( - "autorest --version={0} --use={1} --java --data-plane " - "--java.java-sdks-folder={2} --java.output-folder={3} " - "--java.namespace={4} {5}".format( - autorest, - use, - os.path.abspath(sdk_root), - os.path.abspath(output_dir), - namespace, - " ".join((DPG_ARGUMENTS, input_arguments, security_arguments, artifact_arguments, autorest_options)), - ) - ) - logging.info(command) - if os.system(command) != 0: - logging.error("[GENERATE] Code generation failed. Autorest fails.") - return False - - set_or_default_version(sdk_root, GROUP_ID, module) - update_service_files_for_new_lib(sdk_root, service, GROUP_ID, module) - update_root_pom(sdk_root, service) - # update_version(sdk_root, output_dir) - - return True - - def compile_package(sdk_root: str, group_id: str, module: str) -> bool: command = "mvn --no-transfer-progress clean package -f {0}/pom.xml -Dmaven.javadoc.skip -Dgpg.skip -DskipTestCompile -Djacoco.skip -Drevapi.skip -pl {1}:{2} -am".format( sdk_root, group_id, module @@ -420,190 +140,3 @@ def compile_package(sdk_root: str, group_id: str, module: str) -> bool: ) return False return True - - -def get_generate_parameters(readme_file_abspath: str) -> Tuple[str, str]: - # get parameters from swagger/README.md from sdk repository - - service = None - module = None - if readme_file_abspath: - # try swagger/readme.md for service and module - if not module and os.path.basename(readme_file_abspath).lower() == "readme.md": - dir_name = os.path.dirname(readme_file_abspath).lower() - if os.path.basename(dir_name) == "swagger": - dir_name = os.path.dirname(dir_name) - module = os.path.basename(dir_name) - dir_name = os.path.dirname(dir_name) - service = os.path.basename(dir_name) - dir_name = os.path.dirname(dir_name) - if not os.path.basename(dir_name) == "sdk": - module = None - service = None - - return service, module - - -def uri_file_exists(file_path: str) -> bool: - if file_path.startswith("http://") or file_path.startswith("https://"): - return requests.head(file_path).status_code / 100 == 2 - else: - return os.path.exists(file_path) - - -def uri_file_read(file_path: str) -> str: - if file_path.startswith("http://") or file_path.startswith("https://"): - return requests.get(file_path).text - else: - with open(file_path, "r", encoding="utf-8") as f_in: - return f_in.read() - - -def find_readme(json_file_path: str, readme_file_paths: List[str], spec_root: str) -> Optional[str]: - if not readme_file_paths: - return None - - # ideally we'd like to match readme in more specific path - readme_file_paths = sorted(readme_file_paths, key=len, reverse=True) - - json_dir_name = os.path.dirname(json_file_path) - for readme_file_path in readme_file_paths: - readme_dir_name = os.path.dirname(readme_file_path) - if json_dir_name.startswith(readme_dir_name): - java_readme_path = os.path.join(spec_root, readme_file_path).replace(".md", ".java.md") - if os.path.exists(java_readme_path): - return readme_file_path - - return None - - -def update_readme(sdk_readme_abspath: str, spec_readme: str = None): - # update README_SPEC.md in SDK repo - - with open(sdk_readme_abspath, "r", encoding="utf-8") as f_in: - content = f_in.read() - if content: - yaml_blocks = re.findall(YAML_BLOCK_REGEX, content, re.DOTALL) - for yaml_str in yaml_blocks: - yaml_json = yaml.safe_load(yaml_str) - yaml_json.pop("require", None) - yaml_json.pop("input-file", None) - if spec_readme: - yaml_json["require"] = spec_readme - - # write updated yaml - updated_yaml_str = yaml.dump(yaml_json, width=sys.maxsize, sort_keys=False, Dumper=ListIndentDumper) - - if not yaml_str == updated_yaml_str: - # update readme - updated_content = content.replace(yaml_str, updated_yaml_str, 1) - with open(sdk_readme_abspath, "w", encoding="utf-8") as f_out: - f_out.write(updated_content) - - logging.info( - "[GENERATE] YAML block in README updated from\n{0}\nto\n{1}".format(yaml_str, updated_yaml_str) - ) - break - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument( - "--input-file", - required=False, - help='URL to OpenAPI 2.0 specification JSON as input file. "service" and "module" is required.', - ) - parser.add_argument( - "--spec-readme", - required=False, - help='URL to readme.md from specification repository as input file. "service" and "module" is required.', - ) - parser.add_argument( - "-r", - "--readme", - required=False, - help='URL to "readme.md" as configuration file.', - ) - parser.add_argument( - "--service", - required=False, - help="Service name under sdk/. Sample: storage", - ) - parser.add_argument( - "--module", - required=False, - help="Module name under sdk//. Sample: azure-storage-blob", - ) - parser.add_argument( - "--security", - required=False, - help="Security schemes for authentication. " - 'Sample: "AADToken" for AAD credential for OAuth 2.0 authentication; ' - '"AzureKey" for Azure key credential', - ) - parser.add_argument( - "--security-scopes", - required=False, - help='OAuth 2.0 scopes when "security" includes "AADToken". ' "Sample: https://storage.azure.com/.default", - ) - parser.add_argument( - "--title", - required=False, - help='The name of the client. The name should always ends with "Client". ' - "Sample: BlobClient, which makes BlobClientBuilder as builder class", - ) - parser.add_argument( - "-u", - "--use", - default=AUTOREST_JAVA, - help="Autorest java plugin", - ) - parser.add_argument( - "--autorest", - default=AUTOREST_CORE_VERSION, - help="Autorest version", - ) - parser.add_argument( - "--autorest-options", - default="", - help="Additional autorest options", - ) - - return parser.parse_args() - - -def main(): - args = vars(parse_args()) - - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - - if args["readme"]: - readme_file_abspath = os.path.abspath(args["readme"]) - service, module = get_generate_parameters(readme_file_abspath) - if not module: - raise ValueError("readme.md not found or not well-formed") - args["service"] = service - args["module"] = module - else: - if not args["input_file"] and not args["spec_readme"]: - raise ValueError('Either "readme", or "spec-readme", or "input-file" argument is required') - if not args["service"] or not args["module"]: - raise ValueError('"service" and "module" argument is required') - - succeeded = generate(sdk_root, **args) - if succeeded: - succeeded = compile_package(sdk_root, GROUP_ID, args["module"]) - - if not succeeded: - raise RuntimeError("Failed to generate code or compile the package") - - -if __name__ == "__main__": - logging.basicConfig( - stream=sys.stdout, - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %X", - ) - main() diff --git a/eng/automation/generate_utils.py b/eng/automation/generate_utils.py index 8e7154d1d44e..bcbdf883df07 100755 --- a/eng/automation/generate_utils.py +++ b/eng/automation/generate_utils.py @@ -9,18 +9,13 @@ import requests import tempfile import subprocess -import urllib.parse from typing import Optional, Tuple, List, Union from datetime import datetime pwd = os.getcwd() # os.chdir(os.path.abspath(os.path.dirname(sys.argv[0]))) from parameters import * -from utils import update_service_files_for_new_lib -from utils import update_root_pom -from utils import update_version from utils import is_windows -from utils import set_or_increase_version os.chdir(pwd) @@ -28,85 +23,6 @@ TSPCONFIG_URL_PATTERN = re.compile( r"^https://github.com/(?PAzure/azure-rest-api-specs(-pr)?)/blob/(?P[0-9a-f]{40})/(?P.*)/tspconfig.yaml$", ) -# Add two more indent for list in yaml dump -class ListIndentDumper(yaml.SafeDumper): - - def increase_indent(self, flow=False, indentless=False): - return super(ListIndentDumper, self).increase_indent(flow, False) - - -def generate( - sdk_root: str, - service: str, - spec_root: str, - readme: str, - autorest: str, - use: str, - output_folder: str, - module: str, - namespace: str, - tag: str = None, - version: str = None, - autorest_options: str = "", - premium: bool = False, - **kwargs, -) -> bool: - output_dir = os.path.join( - sdk_root, - "sdk/{0}".format(service), - module, - ) - - require_sdk_integration = not os.path.exists(os.path.join(output_dir, "src")) - - remove_generated_source_code(output_dir, namespace) - - if re.match(r"https?://", spec_root): - readme = urllib.parse.urljoin(spec_root, readme) - else: - readme = os.path.join(spec_root, readme) - - tag_option = "--tag={0}".format(tag) if tag else "" - version_option = "--package-version={0}".format(version) if version else "" - - command = ( - "autorest --version={0} --use={1} --java --java.java-sdks-folder={2} --java.output-folder={3} " - "--java.namespace={4} {5}".format( - autorest, - use, - os.path.abspath(sdk_root), - os.path.abspath(output_dir), - namespace, - " ".join( - ( - tag_option, - version_option, - FLUENTPREMIUM_ARGUMENTS if premium else FLUENTLITE_ARGUMENTS, - autorest_options, - readme, - ) - ), - ) - ) - logging.info(command) - if os.system(command) != 0: - logging.error("[GENERATE] Code generation failed.") - logging.error( - "Please first check if the failure happens only to Java automation, or for all SDK automations. " - "If it happens for all SDK automations, please double check your Swagger, and check whether there is errors in ModelValidation and LintDiff. " - "If it happens to Java alone, you can open an issue to https://github.com/Azure/autorest.java/issues. Please include the link of this Pull Request in the issue." - ) - return False - - group = GROUP_ID - if require_sdk_integration: - update_service_files_for_new_lib(sdk_root, service, group, module) - update_root_pom(sdk_root, service) - update_version(sdk_root, output_folder) - - return True - - def remove_generated_source_code(sdk_folder: str, namespace: str): main_source_folder = os.path.join(sdk_folder, "src/main/java") main_resources_folder = os.path.join(sdk_folder, "src/main/resources") @@ -293,108 +209,6 @@ def get_version( return None -def valid_service(service: str): - return re.sub(r"[^a-z0-9_]", "", service.lower()) - - -def read_api_specs(api_specs_file: str) -> Tuple[str, dict]: - # return comment and api_specs - - with open(api_specs_file, "r", encoding="utf-8") as fin: - lines = fin.readlines() - - comment = "" - - for i, line in enumerate(lines): - if not line.strip().startswith("#"): - comment = "".join(lines[:i]) - api_specs = yaml.safe_load("".join(lines[i:])) - break - else: - raise Exception("api-specs.yml should has non comment line") - - return comment, api_specs - - -def write_api_specs(api_specs_file: str, comment: str, api_specs: dict): - with open(api_specs_file, "w", encoding="utf-8") as fout: - fout.write(comment) - fout.write(yaml.dump(api_specs, width=sys.maxsize, Dumper=ListIndentDumper)) - - -def get_and_update_service_from_api_specs( - api_specs_file: str, - spec: str, - service: str = None, - suffix: str = None, - truncate_service: bool = False, -): - """ - Updates the API specs file with the provided service name and optional suffix. - - Args: - api_specs_file (str): Path to the API specs file. - spec (str): The specification key to update in the API specs. - service (str, optional): The service name to associate with the spec. If not provided, it will be derived. - suffix (str, optional): An optional suffix to add to the spec entry in the API specs file. - truncate_service (bool, optional): Whether to truncate the service name to a maximum length of 32 characters. - - Returns: - str: The validated and potentially updated service name. - """ - special_spec = {"resources"} - if spec in special_spec: - if not service: - service = spec - return valid_service(service) - - comment, api_specs = read_api_specs(api_specs_file) - - api_spec = api_specs.get(spec) - if not service: - if api_spec: - service = api_spec.get("service") - if not service: - service = spec - # remove segment contains ".", e.g. "Microsoft.KubernetesConfiguration", "Astronomer.Astro" - service = re.sub(r"/[^/]+(\.[^/]+)+", "", service) - # truncate length of service to 32, as this is the maximum length for package name in Java repository - if truncate_service: - service = valid_service(service) - max_length = 32 - if len(service) > max_length: - logging.warning(f'[VALIDATE] service name truncated from "{service}" to "{service[:max_length]}"') - service = service[:max_length] - service = valid_service(service) - - if service != spec: - api_specs[spec] = dict() if not api_spec else api_spec - api_specs[spec]["service"] = service - - if suffix: - api_specs[spec]["suffix"] = suffix - - write_api_specs(api_specs_file, comment, api_specs) - - return service - - -def get_suffix_from_api_specs(api_specs_file: str, spec: str): - comment, api_specs = read_api_specs(api_specs_file) - - api_spec = api_specs.get(spec) - if api_spec and api_spec.get("suffix"): - return api_spec.get("suffix") - - return None - - -def update_spec(spec: str, subspec: str) -> str: - if subspec: - spec = spec + subspec - return spec - - def resolve_tspconfig_variables(value: str, config: dict, emitter_name: str) -> Optional[str]: """ Resolve template variables in a tspconfig value, following the same resolution @@ -712,10 +526,6 @@ def clean_sdk_folder(sdk_root: str, sdk_folder: str) -> bool: return succeeded -def is_mgmt_premium(module: str) -> bool: - return module in FLUENT_PREMIUM_PACKAGES - - def copy_file_sync(source: str, target: str) -> None: """ Copy a single file from source to target. diff --git a/eng/automation/generation.yml b/eng/automation/generation.yml index 01fc0daa035b..d2a40319bb77 100644 --- a/eng/automation/generation.yml +++ b/eng/automation/generation.yml @@ -2,15 +2,6 @@ trigger: none pr: none -parameters: # parameters(instead of variables) is used here for template compile time expressions - - name: RELEASE_TYPE - displayName: Release from - type: string - values: - - Swagger - - TypeSpec - default: Swagger - variables: - group: Release Secrets for GitHub - name: MAVEN_CACHE_FOLDER @@ -34,16 +25,10 @@ steps: inputs: versionSpec: '$(NodeVersion)' - - bash: | - npm install -g autorest - displayName: 'Install autorest' - condition: ${{ eq(parameters.RELEASE_TYPE, 'Swagger') }} - - bash: | npm ci workingDirectory: $(Build.SourcesDirectory)/eng/common/tsp-client displayName: 'Install tsp-client' - condition: ${{ eq(parameters.RELEASE_TYPE, 'TypeSpec') }} # - template: /eng/common/testproxy/test-proxy-tool.yml # parameters: @@ -56,25 +41,12 @@ steps: set -x ./eng/automation/generate.py --tsp-config "$(TSP_CONFIG)" --version "$(VERSION)" --auto-commit-external-change --user-name "azure-sdk" --user-email "azuresdk@microsoft.com" displayName: Generation from TypeSpec - condition: ${{ eq(parameters.RELEASE_TYPE, 'TypeSpec') }} - - - bash: | - export PATH=$JAVA_HOME_11_X64/bin:$PATH - java -version - set -x - ./eng/automation/generate.py --readme "$(README)" --tag "$(TAG)" --autorest-options="$(AUTOREST_OPTIONS)" --service "$(SERVICE)" --version "$(VERSION)" --suffix "$(SUFFIX)" --auto-commit-external-change --user-name "azure-sdk" --user-email "azuresdk@microsoft.com" - displayName: Generation from Swagger - condition: ${{ eq(parameters.RELEASE_TYPE, 'Swagger') }} - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: PRBranchName: fluent-lite-generation-$(Build.BuildId) - ${{ if eq(parameters.RELEASE_TYPE, 'TypeSpec') }}: # Have to use parameter here. The ${{}} is compile time expression, it won't resolve against runtime variables. - CommitMsg: '[Automation] Generate Fluent Lite from TypeSpec $(README)' - PRTitle: '[Automation] Generate Fluent Lite from TypeSpec $(README)' - ${{ else }}: - CommitMsg: '[Automation] Generate Fluent Lite from Swagger $(README)#$(TAG)' - PRTitle: '[Automation] Generate Fluent Lite from Swagger $(README)#$(TAG)' + CommitMsg: '[Automation] Generate Fluent Lite from TypeSpec $(README)' + PRTitle: '[Automation] Generate Fluent Lite from TypeSpec $(README)' PRLabels: 'Mgmt - Track 2' OpenAsDraft: '$(DRAFT_PULL_REQUEST)' AuthToken: '' diff --git a/eng/automation/parameters.py b/eng/automation/parameters.py index e21bb3ec6f32..82f0f43899ba 100755 --- a/eng/automation/parameters.py +++ b/eng/automation/parameters.py @@ -1,67 +1,18 @@ #!/usr/bin/env python3 -# changeable parameters -# use update logic in generate.py#update_parameters -# set to None first to guarantee it would be updated -SUFFIX = None - -NAMESPACE_SUFFIX = None -ARTIFACT_SUFFIX = None -NAMESPACE_FORMAT = None -ARTIFACT_FORMAT = None -OUTPUT_FOLDER_FORMAT = None - # Constant parameters MAVEN_HOST = "https://repo1.maven.org/maven2" MAVEN_URL = MAVEN_HOST + "/{group_id}/{artifact_id}/{version}/{artifact_id}-{version}.jar" SDK_ROOT = "../../" # related to file dir -AUTOREST_CORE_VERSION = "3.9.7" -AUTOREST_JAVA = "@autorest/java@4.1.63" DEFAULT_VERSION = "1.0.0-beta.1" GROUP_ID = "com.azure.resourcemanager" -API_SPECS_FILE = "api-specs.yaml" CI_FILE_FORMAT = "sdk/{0}/ci.yml" POM_FILE_FORMAT = "sdk/{0}/pom.xml" -README_FORMAT = "specification/{0}/resource-manager/readme.md" JAR_FORMAT = "sdk/{service}/{artifact_id}/target/{artifact_id}-{version}.jar" CHANGELOG_FORMAT = "sdk/{service}/{artifact_id}/CHANGELOG.md" -MODELERFOUR_ARGUMENTS = "--modelerfour.additional-checks=false --modelerfour.lenient-model-deduplication=true" -FLUENTLITE_ARGUMENTS = "{0} --azure-arm --verbose --sdk-integration --generate-samples --generate-tests --fluent=lite --java.fluent=lite --java.license-header=MICROSOFT_MIT_SMALL".format( - MODELERFOUR_ARGUMENTS -) -FLUENTPREMIUM_ARGUMENTS = ( - "--verbose --generate-samples --fluent --java.fluent --java.license-header=MICROSOFT_MIT_SMALL" -) - -FLUENT_PREMIUM_PACKAGES = ( - "azure-resourcemanager-appplatform", - "azure-resourcemanager-appservice", - "azure-resourcemanager-authorization", - "azure-resourcemanager-cdn", - "azure-resourcemanager-compute", - "azure-resourcemanager-containerinstance", - "azure-resourcemanager-containerregistry", - "azure-resourcemanager-containerservice", - "azure-resourcemanager-cosmos", - "azure-resourcemanager-dns", - "azure-resourcemanager-eventhubs", - "azure-resourcemanager-keyvault", - "azure-resourcemanager-monitor", - "azure-resourcemanager-msi", - "azure-resourcemanager-network", - "azure-resourcemanager-privatedns", - "azure-resourcemanager-redis", - "azure-resourcemanager-resources", - "azure-resourcemanager-search", - "azure-resourcemanager-servicebus", - "azure-resourcemanager-sql", - "azure-resourcemanager-storage", - "azure-resourcemanager-trafficmanager", -) - CI_HEADER = """\ # NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. diff --git a/eng/automation/sdk_generate.py b/eng/automation/sdk_generate.py deleted file mode 100755 index 64525f0ad8ba..000000000000 --- a/eng/automation/sdk_generate.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python3 -import os -import re -import sys -import json -import glob -import logging -import argparse -from typing import List - -pwd = os.getcwd() -os.chdir(os.path.abspath(os.path.dirname(sys.argv[0]))) -from parameters import * -from utils import set_or_increase_version -from generate_data import ( - get_or_update_sdk_readme, - sdk_automation_readme, - update_readme, - sdk_automation_typespec, -) -from generate_utils import ( - compare_with_maven_package, - compile_arm_package, - generate, - get_and_update_service_from_api_specs, - get_suffix_from_api_specs, - update_spec, -) - -os.chdir(pwd) - - -def update_parameters(suffix): - # update changeable parameters in parameters.py - global SUFFIX, NAMESPACE_SUFFIX, ARTIFACT_SUFFIX, NAMESPACE_FORMAT, ARTIFACT_FORMAT, OUTPUT_FOLDER_FORMAT - - SUFFIX = suffix - - NAMESPACE_SUFFIX = ".{0}".format(SUFFIX) if SUFFIX else "" - ARTIFACT_SUFFIX = "-{0}".format(SUFFIX) if SUFFIX else "" - NAMESPACE_FORMAT = "com.azure.resourcemanager.{{0}}{0}".format(NAMESPACE_SUFFIX) - ARTIFACT_FORMAT = "azure-resourcemanager-{{0}}{0}".format(ARTIFACT_SUFFIX) - OUTPUT_FOLDER_FORMAT = "sdk/{{0}}/{0}".format(ARTIFACT_FORMAT) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument( - "--spec-root", - default="https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/", - help="Spec root folder", - ) - parser.add_argument( - "-r", - "--readme", - help='Readme path, Sample: "storage" or "specification/storage/resource-manager/readme.md"', - ) - parser.add_argument("-t", "--tag", help="Specific tag") - parser.add_argument("-v", "--version", help="Specific sdk version") - parser.add_argument( - "-s", - "--service", - help="Service Name if not the same as spec name", - ) - parser.add_argument( - "-u", - "--use", - default=AUTOREST_JAVA, - help="Autorest java plugin", - ) - parser.add_argument( - "--autorest", - default=AUTOREST_CORE_VERSION, - help="Autorest version", - ) - parser.add_argument( - "--autorest-options", - default="", - help="Additional autorest options", - ) - parser.add_argument("--suffix", help="Suffix for namespace and artifact") - parser.add_argument( - "--auto-commit-external-change", - action="store_true", - help="Automatic commit the generated code", - ) - parser.add_argument("--user-name", help="User Name for commit") - parser.add_argument("--user-email", help="User Email for commit") - parser.add_argument( - "config", - nargs="*", - ) - - return parser.parse_args() - - -def codegen_sdk_automation(config: dict) -> List[dict]: - # priority: - # 1. autorestConfig from input - # 2. swagger/README.md in sdk repository that matches readme from input - - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - spec_root = os.path.abspath(config["specFolder"]) - - packages = [] - - # find readme.md in spec repository - if not config["relatedReadmeMdFile"]: - return packages - - readme_file_path = config["relatedReadmeMdFile"] - match = re.search( - r"(specification)?/?([^/]+)/data-plane((?:/[^/]+)*)/readme.md", - readme_file_path, - re.IGNORECASE, - ) - if not match: - logging.info( - "[Skip] readme path:%s does not format as specification/([^/]+)/data-plane((?:/[^/]+)*)/readme.md", - readme_file_path, - ) - return packages - - logging.info("[RESOLVE] README from specification %s", readme_file_path) - sdk_readme_abspath = get_or_update_sdk_readme(config, readme_file_path) - - if sdk_readme_abspath: - spec_readme_abspath = os.path.join(spec_root, readme_file_path) - update_readme(sdk_readme_abspath, spec_readme_abspath) - sdk_automation_readme(sdk_readme_abspath, packages, sdk_root) - - return packages - - -def sdk_automation(input_file: str, output_file: str): - with open(input_file, "r") as fin: - config = json.load(fin) - logging.info(f"sdk_automation input: {config}") - - # typespec - packages = sdk_automation_typespec(config) - # autorest - if not packages: - packages = sdk_automation_autorest(config) - - with open(output_file, "w") as fout: - output = { - "packages": packages, - } - json.dump(output, fout) - - -def sdk_automation_autorest(config: dict) -> List[dict]: - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - api_specs_file = os.path.join(base_dir, API_SPECS_FILE) - - packages = [] - if "relatedReadmeMdFile" not in config or not config["relatedReadmeMdFile"]: - return packages - - readme = config["relatedReadmeMdFile"] - match = re.search( - r"(specification)?/?([^/]+)/resource-manager((?:/[^/]+)*)/readme.md", - readme, - re.IGNORECASE, - ) - if not match: - logging.info("[Skip] readme path does not format as */resource-manager/*/readme.md") - else: - spec = match.group(2) - spec = update_spec(spec, match.group(3)) - service = get_and_update_service_from_api_specs(api_specs_file, spec) - - pre_suffix = SUFFIX - suffix = get_suffix_from_api_specs(api_specs_file, spec) - if suffix is None: - suffix = SUFFIX - update_parameters(suffix) - - # TODO: use specific function to detect tag in "resources" - tag = None - if service == "resources": - with open(os.path.join(config["specFolder"], readme)) as fin: - tag_match = re.search(r"tag: (package-resources-\S+)", fin.read()) - if tag_match: - tag = tag_match.group(1) - else: - tag = "package-resources-2021-01" - - module = ARTIFACT_FORMAT.format(service) - output_folder = OUTPUT_FOLDER_FORMAT.format(service) - namespace = NAMESPACE_FORMAT.format(service) - stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module) - succeeded = generate( - sdk_root, - service, - spec_root=config["specFolder"], - readme=readme, - autorest=AUTOREST_CORE_VERSION, - use=AUTOREST_JAVA, - output_folder=output_folder, - module=module, - namespace=namespace, - tag=tag, - ) - if succeeded: - compile_arm_package(sdk_root, module) - - packages.append( - { - "packageName": "{0}".format(ARTIFACT_FORMAT.format(service)), - "path": [ - output_folder, - CI_FILE_FORMAT.format(service), - POM_FILE_FORMAT.format(service), - "eng/versioning", - "pom.xml", - ], - "readmeMd": [readme], - "artifacts": ["{0}/pom.xml".format(output_folder)] - + [jar for jar in glob.glob("{0}/target/*.jar".format(output_folder))], - "apiViewArtifact": next(iter(glob.glob("{0}/target/*-sources.jar".format(output_folder))), None), - "language": "Java", - "result": "succeeded" if succeeded else "failed", - "packageFolder": output_folder, - } - ) - - update_parameters(pre_suffix) - - if not packages: - # try data-plane codegen - packages = codegen_sdk_automation(config) - for package in packages: - if len(package["path"]) > 0: - package["packageFolder"] = package["path"][0] - - return packages - - -def main(): - args = vars(parse_args()) - - if args.get("config"): - return sdk_automation(args["config"][0], args["config"][1]) - - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - api_specs_file = os.path.join(base_dir, API_SPECS_FILE) - - readme = args["readme"] - match = re.match( - r"specification/([^/]+)/resource-manager/readme.md", - readme, - re.IGNORECASE, - ) - if not match: - spec = readme - readme = "specification/{0}/resource-manager/readme.md".format(spec) - else: - spec = match.group(1) - spec = update_spec(spec, match.group(2)) - - args["readme"] = readme - args["spec"] = spec - - # update_parameters( - # args.get('suffix') or get_suffix_from_api_specs(api_specs_file, spec)) - update_parameters(args.get("suffix")) - service = get_and_update_service_from_api_specs(api_specs_file, spec, args["service"]) - args["service"] = service - module = ARTIFACT_FORMAT.format(service) - stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module, **args) - args["version"] = current_version - output_folder = (OUTPUT_FOLDER_FORMAT.format(service),) - namespace = NAMESPACE_FORMAT.format(service) - succeeded = generate(sdk_root, module=module, output_folder=output_folder, namespace=namespace, **args) - - if succeeded: - succeeded = compile_arm_package(sdk_root, module) - if succeeded: - compare_with_maven_package(sdk_root, GROUP_ID, service, stable_version, current_version, module) - - if args.get("auto_commit_external_change") and args.get("user_name") and args.get("user_email"): - pwd = os.getcwd() - try: - os.chdir(sdk_root) - os.system( - "git add eng/versioning eng/automation pom.xml {0} {1}".format( - CI_FILE_FORMAT.format(service), POM_FILE_FORMAT.format(service) - ) - ) - os.system( - 'git -c user.name={0} -c user.email={1} commit -m "[Automation] External Change"'.format( - args["user_name"], args["user_email"] - ) - ) - finally: - os.chdir(pwd) - - if not succeeded: - raise RuntimeError("Failed to generate code or compile the package") - - -if __name__ == "__main__": - logging.basicConfig( - stream=sys.stdout, - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %X", - ) - main() diff --git a/eng/automation/sdk_init.sh b/eng/automation/sdk_init.sh deleted file mode 100755 index 439e6d7761a8..000000000000 --- a/eng/automation/sdk_init.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -apt-get install python3.8 -apt-get install -y --upgrade python3-pip python3-setuptools - -cat << EOF > $1 -{} -EOF diff --git a/eng/codegen_to_sdk_config.json b/eng/codegen_to_sdk_config.json deleted file mode 100644 index a8f592eb755d..000000000000 --- a/eng/codegen_to_sdk_config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "init": { - "initScript": { - "path": "./eng/automation/sdk_init.sh" - } - }, - "generateAndBuild": { - "generateAndBuildScript": { - "path": "./eng/automation/sdk_generate.py", - "script": "python3" - } - } -} \ No newline at end of file From 12dc522d219700abf274a21c9ac117269112239c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 03:41:56 +0000 Subject: [PATCH 3/6] Fix comment typo in parameters.py (related -> relative) Co-authored-by: XiaofeiCao <92354331+XiaofeiCao@users.noreply.github.com> --- eng/automation/parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/automation/parameters.py b/eng/automation/parameters.py index 82f0f43899ba..8d6a2bf4cff6 100755 --- a/eng/automation/parameters.py +++ b/eng/automation/parameters.py @@ -4,7 +4,7 @@ MAVEN_HOST = "https://repo1.maven.org/maven2" MAVEN_URL = MAVEN_HOST + "/{group_id}/{artifact_id}/{version}/{artifact_id}-{version}.jar" -SDK_ROOT = "../../" # related to file dir +SDK_ROOT = "../../" # relative to file dir DEFAULT_VERSION = "1.0.0-beta.1" GROUP_ID = "com.azure.resourcemanager" From fe70576f17fa76cabc508d17bd6735986a161458 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 04:03:46 +0000 Subject: [PATCH 4/6] Remove unused generation.yml pipeline and its manual --tsp-config path in generate.py Co-authored-by: XiaofeiCao <92354331+XiaofeiCao@users.noreply.github.com> --- eng/automation/generate.py | 66 ++--------------------------------- eng/automation/generation.yml | 52 --------------------------- 2 files changed, 2 insertions(+), 116 deletions(-) delete mode 100644 eng/automation/generation.yml diff --git a/eng/automation/generate.py b/eng/automation/generate.py index f5dd91b3747c..4e2ae9eea88f 100755 --- a/eng/automation/generate.py +++ b/eng/automation/generate.py @@ -34,21 +34,6 @@ def parse_args() -> (argparse.ArgumentParser, argparse.Namespace): parser = argparse.ArgumentParser() - parser.add_argument( - "-c", - "--tsp-config", - help="The top level directory where the tspconfig.yaml for the service lives. " - "Currently only support remote url with specific commitID " - "e.g. https://github.com/Azure/azure-rest-api-specs/blob/042e4045dedff4baaf5ae551bf6c8087fbdacd40/specification/deviceregistry/DeviceRegistry.Management/tspconfig.yaml", - ) - parser.add_argument("-v", "--version", help="Specific sdk version") - parser.add_argument( - "--auto-commit-external-change", - action="store_true", - help="Automatic commit the generated code", - ) - parser.add_argument("--user-name", help="User Name for commit") - parser.add_argument("--user-email", help="User Email for commit") parser.add_argument( "config", nargs="*", @@ -336,55 +321,8 @@ def main(): if args.get("config"): return sdk_automation(args["config"][0], args["config"][1]) - base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - sdk_root = os.path.abspath(os.path.join(base_dir, SDK_ROOT)) - - if not args.get("tsp_config"): - parser.print_help() - sys.exit(0) - - tsp_config = args["tsp_config"] - - succeeded, require_sdk_integration, sdk_folder, service, module = generate_typespec_project( - tsp_project=tsp_config, sdk_root=sdk_root, remove_before_regen=True, group_id=GROUP_ID, **args - ) - - stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module, **args) - args["version"] = current_version - - if require_sdk_integration: - update_service_files_for_new_lib(sdk_root, service, GROUP_ID, module) - update_root_pom(sdk_root, service) - - output_folder = sdk_folder - update_version(sdk_root, output_folder) - update_changelog_version(sdk_root, output_folder, current_version) - - if succeeded: - succeeded = compile_arm_package(sdk_root, module) - if succeeded: - latest_release_version = get_latest_release_version(stable_version, current_version) - compare_with_maven_package(sdk_root, GROUP_ID, service, latest_release_version, current_version, module) - - if args.get("auto_commit_external_change") and args.get("user_name") and args.get("user_email"): - pwd = os.getcwd() - try: - os.chdir(sdk_root) - os.system( - "git add eng/versioning eng/automation pom.xml {0} {1}".format( - CI_FILE_FORMAT.format(service), POM_FILE_FORMAT.format(service) - ) - ) - os.system( - 'git -c user.name={0} -c user.email={1} commit -m "[Automation] External Change"'.format( - args["user_name"], args["user_email"] - ) - ) - finally: - os.chdir(pwd) - - if not succeeded: - raise RuntimeError("Failed to generate code or compile the package") + parser.print_help() + sys.exit(0) if __name__ == "__main__": diff --git a/eng/automation/generation.yml b/eng/automation/generation.yml deleted file mode 100644 index d2a40319bb77..000000000000 --- a/eng/automation/generation.yml +++ /dev/null @@ -1,52 +0,0 @@ -trigger: none - -pr: none - -variables: - - group: Release Secrets for GitHub - - name: MAVEN_CACHE_FOLDER - value: $(Pipeline.Workspace)/.m2/repository - - name: MAVEN_OPTS - value: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' - - name: NodeVersion - value: '20.x' - - name: JavaVersion - value: '17' - -steps: - - bash: | - sudo apt-get install -y --upgrade python3-pip python3-setuptools - pip3 install --upgrade wheel - pip3 install --upgrade PyYAML requests - displayName: Update python - - - task: NodeTool@0 - displayName: 'Install Node.js $(NodeVersion)' - inputs: - versionSpec: '$(NodeVersion)' - - - bash: | - npm ci - workingDirectory: $(Build.SourcesDirectory)/eng/common/tsp-client - displayName: 'Install tsp-client' - - # - template: /eng/common/testproxy/test-proxy-tool.yml - # parameters: - # runProxy: true - # targetVersion: 1.0.0-dev.20230908.1 - - - bash: | - export PATH=$JAVA_HOME_11_X64/bin:$PATH - java -version - set -x - ./eng/automation/generate.py --tsp-config "$(TSP_CONFIG)" --version "$(VERSION)" --auto-commit-external-change --user-name "azure-sdk" --user-email "azuresdk@microsoft.com" - displayName: Generation from TypeSpec - - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - PRBranchName: fluent-lite-generation-$(Build.BuildId) - CommitMsg: '[Automation] Generate Fluent Lite from TypeSpec $(README)' - PRTitle: '[Automation] Generate Fluent Lite from TypeSpec $(README)' - PRLabels: 'Mgmt - Track 2' - OpenAsDraft: '$(DRAFT_PULL_REQUEST)' - AuthToken: '' From 896383d09124bef314302b4cbb1aa519249ad5af Mon Sep 17 00:00:00 2001 From: "Xiaofei Cao (from Dev Box)" Date: Mon, 22 Jun 2026 10:43:06 +0800 Subject: [PATCH 5/6] Remove unused eng/automation/changelog.py standalone script Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/automation/changelog.py | 48 ------------------------------------- 1 file changed, 48 deletions(-) delete mode 100755 eng/automation/changelog.py diff --git a/eng/automation/changelog.py b/eng/automation/changelog.py deleted file mode 100755 index cd08d5ebcc25..000000000000 --- a/eng/automation/changelog.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import logging -import argparse - -pwd = os.getcwd() -os.chdir(os.path.abspath(os.path.dirname(sys.argv[0]))) -from parameters import * -from generate import update_parameters -from generate_utils import get_version -from generate_utils import compile_arm_package -from generate import compare_with_maven_package - -os.chdir(pwd) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("-s", "--service", required=True) - parser.add_argument("--suffix") - parser.add_argument("-c", "--compile", action="store_true") - return parser.parse_args() - - -def main(): - args = vars(parse_args()) - sdk_root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), SDK_ROOT)) - service = args["service"] - update_parameters(args.get("suffix")) - - if args.get("compile"): - compile_arm_package(sdk_root, service) - - versions = get_version(sdk_root, GROUP_ID, service).split(";") - stable_version = versions[1] - current_version = versions[2] - compare_with_maven_package(sdk_root, GROUP_ID, service, stable_version, current_version) - - -if __name__ == "__main__": - logging.basicConfig( - stream=sys.stdout, - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %X", - ) - main() From 9122a99562410cd2e0be2feb21e09d85b28bbc02 Mon Sep 17 00:00:00 2001 From: "Xiaofei Cao (from Dev Box)" Date: Mon, 22 Jun 2026 10:53:41 +0800 Subject: [PATCH 6/6] Require exactly two config arguments in generate.py main() Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/automation/generate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/automation/generate.py b/eng/automation/generate.py index 4e2ae9eea88f..70fdae0bdc9c 100755 --- a/eng/automation/generate.py +++ b/eng/automation/generate.py @@ -318,8 +318,11 @@ def main(): (parser, args) = parse_args() args = vars(args) - if args.get("config"): - return sdk_automation(args["config"][0], args["config"][1]) + config = args.get("config") + if config: + if len(config) != 2: + parser.error("config requires exactly two arguments: generationInput and generationOutput") + return sdk_automation(config[0], config[1]) parser.print_help() sys.exit(0)