diff --git a/backend/Models/MethodsTemplate.py b/backend/Models/MethodsTemplate.py index f3d5af8..ed37b83 100644 --- a/backend/Models/MethodsTemplate.py +++ b/backend/Models/MethodsTemplate.py @@ -1,24 +1,40 @@ from pydantic import BaseModel, Field, RootModel from typing import Dict, Any, Optional, Union + class ParameterInfo(BaseModel): """Schema for parameter information""" + type: str = Field(..., description="Type annotation of the parameter") value: Any = Field(..., description="Default or required value of the parameter") desc: str = Field(..., description="Description of the parameter") + class MethodTemplate(BaseModel): """Schema for individual method template""" + method_name: str = Field(..., description="Name of the method") - module_path: str = Field(..., description="Full path to the module containing the method") + module_path: str = Field( + ..., description="Full path to the module containing the method" + ) method_desc: str = Field(..., description="Description of what the method does") method_doc: str = Field(..., description="Link to Docs for the method") - parameters: Dict[str, ParameterInfo] = Field(..., description="Dictionary of parameter information") + parameters: Dict[str, ParameterInfo] = Field( + ..., description="Dictionary of parameter information" + ) + class ModuleTemplates(RootModel): """Schema for all methods in a module""" - root: Dict[str, MethodTemplate] = Field(..., description="Dictionary of method templates in a module") + + root: Dict[str, MethodTemplate] = Field( + ..., description="Dictionary of method templates in a module" + ) + class AllTemplates(RootModel): """Schema for all modules and their methods""" - root: Dict[str, Dict[str, MethodTemplate]] = Field(..., description="Dictionary of all modules and their method templates") + + root: Dict[str, Dict[str, MethodTemplate]] = Field( + ..., description="Dictionary of all modules and their method templates" + ) diff --git a/backend/Models/YamlModels.py b/backend/Models/YamlModels.py index 234363b..58cfdfb 100644 --- a/backend/Models/YamlModels.py +++ b/backend/Models/YamlModels.py @@ -1,11 +1,13 @@ from pydantic import BaseModel -from typing import Dict, Optional,List,Any +from typing import Dict, Optional, List, Any + class SweepConfig(BaseModel): methodId: str paramName: str sweepType: str + class YamlGenerateRequest(BaseModel): data: List[Dict[str, Any]] fileName: str diff --git a/backend/main.py b/backend/main.py index 3e37db2..4a6081e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -16,8 +16,6 @@ ) - app.include_router(methods_router) app.include_router(yaml_router) app.include_router(proxy_router) - diff --git a/backend/routers/methods.py b/backend/routers/methods.py index a754f91..2d35f94 100644 --- a/backend/routers/methods.py +++ b/backend/routers/methods.py @@ -1,5 +1,5 @@ from utils.methods import METHOD_CATEGORIES -from fastapi import HTTPException,APIRouter +from fastapi import HTTPException, APIRouter from utils.generator import generate_method_template from typing import Dict, List from Models.MethodsTemplate import AllTemplates @@ -11,6 +11,7 @@ tags=["methods"], ) + def get_methods_templates(module_methods: Dict[str, List[str]]) -> Dict: """ Helper function to generate templates for a given module and its methods @@ -39,13 +40,16 @@ async def endpoint(): return AllTemplates(root=result) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + return endpoint + # Register all category endpoints for category in METHOD_CATEGORIES.keys(): endpoint = create_category_endpoint(category) methods_router.get(f"/{category}", response_model=AllTemplates)(endpoint) + def filter_pipelines_with_tomopy(pipelines: dict) -> dict: """ Filters out pipelines where any method has a module_path starting with 'tomopy.'. @@ -68,6 +72,7 @@ def filter_pipelines_with_tomopy(pipelines: dict) -> dict: return filtered_pipelines + @methods_router.get("/fullpipelines") async def get_full_pipelines(): try: @@ -75,6 +80,7 @@ async def get_full_pipelines(): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + # Main endpoint for all methods @methods_router.get("/", response_model=AllTemplates) async def get_all_methods(): @@ -86,4 +92,3 @@ async def get_all_methods(): return AllTemplates(root=all_templates) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - \ No newline at end of file diff --git a/backend/routers/proxy.py b/backend/routers/proxy.py index c2b888b..51fb73e 100644 --- a/backend/routers/proxy.py +++ b/backend/routers/proxy.py @@ -11,6 +11,7 @@ proxy_router = APIRouter(prefix="/proxy", tags=["proxy"]) + @proxy_router.get("/tiff") async def proxy_tiff(url: str = Query(..., description="The S3 URL to proxy")): """ @@ -18,27 +19,31 @@ async def proxy_tiff(url: str = Query(..., description="The S3 URL to proxy")): """ try: logger.info(f"Proxying TIFF file from URL: {url}") - + # Validate that it's a reasonable URL (basic security) if not url.startswith(("https://", "http://")): raise HTTPException(status_code=400, detail="Invalid URL scheme") - + # Use httpx to fetch the file async with httpx.AsyncClient(timeout=60.0) as client: response = await client.get(url) - + if response.status_code != 200: - logger.error(f"Failed to fetch file: {response.status_code} - {response.text}") + logger.error( + f"Failed to fetch file: {response.status_code} - {response.text}" + ) raise HTTPException( - status_code=response.status_code, - detail=f"Failed to fetch file: {response.status_code}" + status_code=response.status_code, + detail=f"Failed to fetch file: {response.status_code}", ) - + # Get the content type from the original response content_type = response.headers.get("content-type", "image/tiff") - - logger.info(f"Successfully fetched file, size: {len(response.content)} bytes") - + + logger.info( + f"Successfully fetched file, size: {len(response.content)} bytes" + ) + # Return the file content with appropriate headers return Response( content=response.content, @@ -48,27 +53,31 @@ async def proxy_tiff(url: str = Query(..., description="The S3 URL to proxy")): "Access-Control-Allow-Methods": "GET", "Access-Control-Allow-Headers": "*", "Content-Length": str(len(response.content)), - "Cache-Control": "public, max-age=3600" # Cache for 1 hour - } + "Cache-Control": "public, max-age=3600", # Cache for 1 hour + }, ) - + except httpx.TimeoutException: logger.error(f"Timeout while fetching URL: {url}") raise HTTPException(status_code=504, detail="Timeout while fetching file") - + except httpx.HTTPError as e: logger.error(f"HTTP error while fetching URL {url}: {str(e)}") raise HTTPException(status_code=502, detail=f"Error fetching file: {str(e)}") - + except Exception as e: logger.error(f"Unexpected error while proxying URL {url}: {str(e)}") raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") + @proxy_router.get("/tiff-pages") async def proxy_tiff_pages( url: str = Query(..., description="The S3 URL to proxy"), - page: int = Query(None, description="Page number to extract (0-based). If not provided, returns metadata."), - downsample_rate: int = 1 + page: int = Query( + None, + description="Page number to extract (0-based). If not provided, returns metadata.", + ), + downsample_rate: int = 1, ): """ Process multi-page TIFF files server-side using Pillow. @@ -77,27 +86,27 @@ async def proxy_tiff_pages( """ try: logger.info(f"Processing TIFF pages from URL: {url}, page: {page}") - + # Validate URL if not url.startswith(("https://", "http://")): raise HTTPException(status_code=400, detail="Invalid URL scheme") - + # Fetch the TIFF file async with httpx.AsyncClient(timeout=60.0) as client: response = await client.get(url) - + if response.status_code != 200: logger.error(f"Failed to fetch TIFF file: {response.status_code}") raise HTTPException( - status_code=response.status_code, - detail=f"Failed to fetch TIFF file: {response.status_code}" + status_code=response.status_code, + detail=f"Failed to fetch TIFF file: {response.status_code}", ) - + logger.info(f"Fetched TIFF file, size: {len(response.content)} bytes") - + # Process TIFF with Pillow tiff_data = BytesIO(response.content) - + with Image.open(tiff_data) as img: # If no page specified, return metadata if page is None: @@ -109,55 +118,63 @@ async def proxy_tiff_pages( page_count += 1 except EOFError: pass - + # Get dimensions from first page img.seek(0) width, height = img.size - + metadata = { "page_count": page_count, "width": width, "height": height, "format": img.format, - "mode": img.mode + "mode": img.mode, } - + logger.info(f"TIFF metadata: {metadata}") - + return JSONResponse( content=metadata, headers={ "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "GET", "Access-Control-Allow-Headers": "*", - "Cache-Control": "public, max-age=3600" - } + "Cache-Control": "public, max-age=3600", + }, ) - + # Extract specific page else: try: img.seek(page) if downsample_rate != 1: width, height = img.size - img = img.resize((width // downsample_rate, height // downsample_rate)) + img = img.resize( + (width // downsample_rate, height // downsample_rate) + ) logger.info(f"Extracting page {page}, size: {img.size}") - + # Convert to RGB if necessary (TIFF might be in different modes) - if img.mode not in ('RGB', 'RGBA'): - if img.mode == 'I;16': + if img.mode not in ("RGB", "RGBA"): + if img.mode == "I;16": array = np.array(img) - normalized = (array.astype(np.uint16) - array.min()) * 255.0 / (array.max() - array.min()) + normalized = ( + (array.astype(np.uint16) - array.min()) + * 255.0 + / (array.max() - array.min()) + ) img = Image.fromarray(normalized.astype(np.uint8)) - img = img.convert('RGB') - + img = img.convert("RGB") + # Save as PNG to BytesIO png_buffer = BytesIO() - img.save(png_buffer, format='PNG', optimize=True) + img.save(png_buffer, format="PNG", optimize=True) png_data = png_buffer.getvalue() - - logger.info(f"Converted page {page} to PNG, size: {len(png_data)} bytes") - + + logger.info( + f"Converted page {page} to PNG, size: {len(png_data)} bytes" + ) + return Response( content=png_data, media_type="image/png", @@ -166,25 +183,29 @@ async def proxy_tiff_pages( "Access-Control-Allow-Methods": "GET", "Access-Control-Allow-Headers": "*", "Content-Length": str(len(png_data)), - "Cache-Control": "public, max-age=3600" - } + "Cache-Control": "public, max-age=3600", + }, ) - + except EOFError: logger.error(f"Page {page} does not exist in TIFF") - raise HTTPException(status_code=404, detail=f"Page {page} not found") + raise HTTPException( + status_code=404, detail=f"Page {page} not found" + ) except Exception as e: logger.error(f"Error processing page {page}: {str(e)}") - raise HTTPException(status_code=500, detail=f"Error processing page: {str(e)}") - + raise HTTPException( + status_code=500, detail=f"Error processing page: {str(e)}" + ) + except httpx.TimeoutException: logger.error(f"Timeout while fetching URL: {url}") raise HTTPException(status_code=504, detail="Timeout while fetching file") - + except httpx.HTTPError as e: logger.error(f"HTTP error while fetching URL {url}: {str(e)}") raise HTTPException(status_code=502, detail=f"Error fetching file: {str(e)}") - + except Exception as e: logger.error(f"Unexpected error processing TIFF: {str(e)}") - raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") \ No newline at end of file + raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") diff --git a/backend/routers/yaml.py b/backend/routers/yaml.py index 4dfa899..2203a09 100644 --- a/backend/routers/yaml.py +++ b/backend/routers/yaml.py @@ -10,48 +10,51 @@ tags=["yaml"], ) + @yaml_router.post("/generate") async def generate_yaml(request: YamlGenerateRequest): try: # Convert data to YAML - yaml_content = yaml.dump(request.data, sort_keys=False, default_flow_style=False) - + yaml_content = yaml.dump( + request.data, sort_keys=False, default_flow_style=False + ) + # Post-process the YAML string to add custom tags if needed if request.sweepConfig: method_id = request.sweepConfig.methodId param_name = request.sweepConfig.paramName sweep_type = request.sweepConfig.sweepType tag = "!SweepRange" if sweep_type == "range" else "!Sweep" - + # Split YAML content into entries (each method is a separate entry) - yaml_entries = yaml_content.split('- method:') + yaml_entries = yaml_content.split("- method:") header = yaml_entries[0] # Store any header content - entries = ['- method:' + entry for entry in yaml_entries[1:]] # Restore the "- method:" prefix - + entries = [ + "- method:" + entry for entry in yaml_entries[1:] + ] # Restore the "- method:" prefix + # Process each entry for i, entry in enumerate(entries): # Check if this is the target method - if f"method: {method_id}" in entry.split('\n')[0]: + if f"method: {method_id}" in entry.split("\n")[0]: # Replace parameter in this entry only param_pattern = f"(\\s+{param_name}:)(\\s+)" entries[i] = re.sub(param_pattern, f"\\1 {tag}\\2", entry) break - + # Reconstruct the YAML content - yaml_content = header + ''.join(entries) - + yaml_content = header + "".join(entries) + # Set the response headers for file download filename = f"{request.fileName}.yaml" headers = { - 'Content-Disposition': f'attachment; filename="{filename}"', - 'Content-Type': 'application/x-yaml', + "Content-Disposition": f'attachment; filename="{filename}"', + "Content-Type": "application/x-yaml", } - + return Response( - content=yaml_content, - media_type='application/x-yaml', - headers=headers + content=yaml_content, media_type="application/x-yaml", headers=headers ) - + except Exception as e: - raise HTTPException(status_code=500, detail=f"Error generating YAML: {str(e)}") \ No newline at end of file + raise HTTPException(status_code=500, detail=f"Error generating YAML: {str(e)}") diff --git a/backend/utils/generator.py b/backend/utils/generator.py index a7f3b53..7c69a88 100644 --- a/backend/utils/generator.py +++ b/backend/utils/generator.py @@ -9,7 +9,6 @@ import importlib from typing import Dict, Union - def _get_discard_params() -> List[str]: """ @@ -45,7 +44,8 @@ def _get_discard_params() -> List[str]: ] return discard_params -def _get_mustchange_params() ->List[str]: + +def _get_mustchange_params() -> List[str]: mustchange_params = [ "proj1", "proj2", @@ -53,13 +53,14 @@ def _get_mustchange_params() ->List[str]: "asynchronous", "center", "glob_stats", - "overlap" + "overlap", ] return mustchange_params + def set_param_value(name: str): - if name in ["proj1", "proj2","axis"]: - return "auto" + if name in ["proj1", "proj2", "axis"]: + return "auto" elif name == "kwargs": # params_dict["#additional parameters"] = "AVAILABLE" # parsing hashtag to yaml comes with quotes, for now we simply ignore the field @@ -76,163 +77,177 @@ def set_param_value(name: str): else: return "REQUIRED" + def generate_method_template(module_name: str, method_name: str) -> Dict: """ Generate a comprehensive method template with detailed parameter information. - + Parameters ---------- module_name : str The name of the module containing the method method_name : str The name of the method to analyze - + Returns ------- Dict A dictionary containing method details and parameter information """ - - # Getting the discarded params list + + # Getting the discarded params list discard_params = _get_discard_params() mustchange_params = _get_mustchange_params() # Import the module imported_module = importlib.import_module(str(module_name)) - - # Get the method + + # Get the method method = getattr(imported_module, method_name) - + # Get method signature and docstring method_signature = inspect.signature(method) method_docstring = inspect.getdoc(method) or "" - + # Parse docstring for parameter descriptions docstring_params = parse_docstring(method_docstring) # Prepare parameters dictionary parameters_dict = {} - + for name, param in method_signature.parameters.items(): if name not in discard_params: - + # Get parameter type param_type = "Any" if param.annotation != inspect.Parameter.empty: param_type = _convert_type_to_string(param.annotation) - + # Get default value default_value = ( - "REQUIRED" if (param.default == inspect.Parameter.empty and name not in mustchange_params) else - param.default if (name not in mustchange_params) + "REQUIRED" + if ( + param.default == inspect.Parameter.empty + and name not in mustchange_params + ) + else ( + param.default + if (name not in mustchange_params) else set_param_value(name) + ) ) - + # Get parameter description from docstring param_desc = "" - if 'parameters' in docstring_params and name in docstring_params['parameters']: - param_info = docstring_params['parameters'].get(name, {}) - param_desc = param_info.get('description', '') + if ( + "parameters" in docstring_params + and name in docstring_params["parameters"] + ): + param_info = docstring_params["parameters"].get(name, {}) + param_desc = param_info.get("description", "") # Create parameter entry parameters_dict[name] = { "type": param_type, "value": default_value, "desc": param_desc, - } - + # Construct method dictionary - if(module_name.split(".")[0]=="httomolib"): - linkToDoc = f"https://diamondlightsource.github.io/httomolib/api/{module_name}.html" + if module_name.split(".")[0] == "httomolib": + linkToDoc = ( + f"https://diamondlightsource.github.io/httomolib/api/{module_name}.html" + ) else: - linkToDoc = f"https://diamondlightsource.github.io/httomolibgpu/api/{module_name}.html" - + linkToDoc = ( + f"https://diamondlightsource.github.io/httomolibgpu/api/{module_name}.html" + ) + method_dict = { "method_name": method_name, "module_path": module_name, - "method_desc" :docstring_params["desc"], - "method_doc" : linkToDoc, - "parameters": parameters_dict + "method_desc": docstring_params["desc"], + "method_doc": linkToDoc, + "parameters": parameters_dict, } - + return method_dict + import re + def parse_docstring(docstring): """ Parse a docstring to extract description and parameter information. - + Parameters ---------- docstring : str The docstring to parse - + Returns ------- Dict A dictionary containing description and parameter details """ # Remove citations in the format :cite:`something` - docstring = re.sub(r':cite:`[^`]+`', '', docstring) - + docstring = re.sub(r":cite:`[^`]+`", "", docstring) + # Split the docstring into sections - sections = re.split(r'\n(Parameters|Raises|Returns)\n-+', docstring) - + sections = re.split(r"\n(Parameters|Raises|Returns)\n-+", docstring) + # Clean and extract first sentence - description_text = sections[0].strip().replace('\n', ' ') - + description_text = sections[0].strip().replace("\n", " ") + # Match the first sentence (ending with a period) - first_sentence_match = re.match(r'^(.*?\.)(?:\s|$)', description_text) - description = first_sentence_match.group(1).strip() if first_sentence_match else description_text - + first_sentence_match = re.match(r"^(.*?\.)(?:\s|$)", description_text) + description = ( + first_sentence_match.group(1).strip() + if first_sentence_match + else description_text + ) + # Initialize parameters dictionary parameters = {} - + # If there's a Parameters section - if len(sections) > 1 and 'Parameters' in sections: + if len(sections) > 1 and "Parameters" in sections: # Find the index of the Parameters section - param_index = sections.index('Parameters') + 1 - + param_index = sections.index("Parameters") + 1 + # Split parameters section into lines - param_lines = sections[param_index].strip().split('\n') - + param_lines = sections[param_index].strip().split("\n") + # Regex to match parameter definitions - param_pattern = re.compile(r'^(\w+)\s*:\s*(.+)') - + param_pattern = re.compile(r"^(\w+)\s*:\s*(.+)") + current_param = None for line in param_lines: stripped = line.strip() - + # Check if line is a new parameter definition param_match = param_pattern.match(stripped) if param_match: current_param = param_match.group(1) param_type = param_match.group(2) - parameters[current_param] = { - 'type': param_type, - 'description': '' - } + parameters[current_param] = {"type": param_type, "description": ""} elif current_param: # Avoid adding lines that look like new parameter definitions - if not re.match(r'^\w+\s*:', stripped) and stripped: + if not re.match(r"^\w+\s*:", stripped) and stripped: # Add to description, preserving first line if empty - if not parameters[current_param]['description']: - parameters[current_param]['description'] = stripped + if not parameters[current_param]["description"]: + parameters[current_param]["description"] = stripped else: - parameters[current_param]['description'] += ' ' + stripped - - return { - 'desc': description, - 'parameters': parameters - } + parameters[current_param]["description"] += " " + stripped + + return {"desc": description, "parameters": parameters} def _convert_type_to_string(type_annotation): """Convert type annotation to a readable string representation""" # Handle Optional[X] which is Union[X, NoneType] - if hasattr(type_annotation, '__origin__') and type_annotation.__origin__ is Union: + if hasattr(type_annotation, "__origin__") and type_annotation.__origin__ is Union: args = type_annotation.__args__ # Check if Union contains NoneType (Optional case) if len(args) == 2 and type(None) in args: @@ -243,13 +258,13 @@ def _convert_type_to_string(type_annotation): return f"Union[{', '.join(_convert_type_to_string(arg) for arg in args)}]" # Handle complex generic types like List, Dict, etc. - if hasattr(type_annotation, '__origin__'): + if hasattr(type_annotation, "__origin__"): origin = type_annotation.__origin__.__name__ args = [_convert_type_to_string(arg) for arg in type_annotation.__args__] return f"{origin}[{', '.join(args)}]" # Handle standard types - if hasattr(type_annotation, '__name__'): + if hasattr(type_annotation, "__name__"): return type_annotation.__name__ # Fallback for unknown annotations diff --git a/backend/utils/methods.py b/backend/utils/methods.py index 8574d64..5286695 100644 --- a/backend/utils/methods.py +++ b/backend/utils/methods.py @@ -1,25 +1,23 @@ METHOD_CATEGORIES = { "denoising-artefactsremoval": { - "httomolibgpu.misc.corr": ["remove_outlier","median_filter"], - "httomolibgpu.misc.denoise": ["total_variation_PD","total_variation_ROF"] + "httomolibgpu.misc.corr": ["remove_outlier", "median_filter"], + "httomolibgpu.misc.denoise": ["total_variation_PD", "total_variation_ROF"], }, "image-saving": { "httomolib.misc.images": ["save_to_images"], - "httomolibgpu.misc.rescale": ["rescale_to_int"] - }, - "segmentation": { - "httomolib.misc.segm": ["binary_thresholding"] + "httomolibgpu.misc.rescale": ["rescale_to_int"], }, + "segmentation": {"httomolib.misc.segm": ["binary_thresholding"]}, "morphological": { "httomolib.misc.morph": ["data_reducer"], - "httomolibgpu.misc.morph": ["sino_360_to_180", "data_resampler"] + "httomolibgpu.misc.morph": ["sino_360_to_180", "data_resampler"], }, "normalization": { "httomolibgpu.prep.normalize": ["dark_flat_field_correction", "minus_log"] }, "phase-retrieval": { "httomolib.prep.phase": ["paganin_filter"], - "httomolibgpu.prep.phase": ["paganin_filter_savu_legacy", "paganin_filter"] + "httomolibgpu.prep.phase": ["paganin_filter_savu_legacy", "paganin_filter"], }, "stripe-removal": { "httomolibgpu.prep.stripe": [ @@ -27,7 +25,7 @@ "remove_stripe_fw", "remove_stripe_ti", "remove_all_stripe", - "raven_filter" + "raven_filter", ] }, "distortion-correction": { @@ -37,10 +35,18 @@ "httomolibgpu.recon.rotation": [ "find_center_vo", "find_center_360", - "find_center_pc" + "find_center_pc", ] }, "reconstruction": { - "httomolibgpu.recon.algorithm": ["FBP3d_tomobar", "SIRT3d_tomobar", "CGLS3d_tomobar", "LPRec3d_tomobar","FBP2d_astra", "FISTA3d_tomobar", "ADMM3d_tomobar"] + "httomolibgpu.recon.algorithm": [ + "FBP3d_tomobar", + "SIRT3d_tomobar", + "CGLS3d_tomobar", + "LPRec3d_tomobar", + "FBP2d_astra", + "FISTA3d_tomobar", + "ADMM3d_tomobar", + ] }, }