Implemented first instance of preview

Author: m-messer

evaluation_function/preview.py

@@ -1,28 +1,56 @@
+import ast
 from typing import Any
 from lf_toolkit.preview import Result, Params, Preview
 
-def preview_function(response: Any, params: Params) -> Result:
-    """
-    Function used to preview a student response.
-    ---
-    The handler function passes three arguments to preview_function():
+_BLOCKED_MODULES = {
+    "os", "sys", "subprocess", "socket", "urllib", "http",
+    "requests", "shutil", "pathlib", "ftplib", "smtplib",
+    "ctypes", "multiprocessing", "threading", "importlib",
+    "pickle", "builtins",
+}
+
+_BLOCKED_BUILTINS = {"exec", "eval", "compile", "open", "__import__", "input"}
+
+
+class _SecurityVisitor(ast.NodeVisitor):
+    def __init__(self):
+        self.violations: list[str] = []
 
-    - `response` which are the answers provided by the student.
-    - `params` which are any extra parameters that may be useful,
-        e.g., error tolerances.
+    def visit_Import(self, node):
+        for alias in node.names:
+            root = alias.name.split(".")[0]
+            if root in _BLOCKED_MODULES:
+                self.violations.append(f"import of '{root}' is not allowed")
+        self.generic_visit(node)
 
-    The output of this function is what is returned as the API response
-    and therefore must be JSON-encodable. It must also conform to the
-    response schema.
+    def visit_ImportFrom(self, node):
+        if node.module:
+            root = node.module.split(".")[0]
+            if root in _BLOCKED_MODULES:
+                self.violations.append(f"import of '{root}' is not allowed")
+        self.generic_visit(node)
 
-    Any standard python library may be used, as well as any package
-    available on pip (provided it is added to requirements.txt).
+    def visit_Call(self, node):
+        if isinstance(node.func, ast.Name) and node.func.id in _BLOCKED_BUILTINS:
+            self.violations.append(f"use of '{node.func.id}()' is not allowed")
+        self.generic_visit(node)
 
-    The way you wish to structure you code (all in this function, or
-    split into many) is entirely up to you.
-    """
+    def visit_Attribute(self, node):
+        if node.attr.startswith("__") and node.attr.endswith("__"):
+            self.violations.append(f"access to '{node.attr}' is not allowed")
+        self.generic_visit(node)
 
+
+def preview_function(response: Any, params: Params) -> Result:
     try:
-        return Result(preview=Preview(sympy=response))
-    except Exception as e:
-        return Result(preview=Preview(feedback=str(e)))
+        tree = ast.parse(str(response))
+    except SyntaxError as e:
+        return Result(preview=Preview(feedback=f"SyntaxError: {e.msg} (line {e.lineno})"))
+
+    visitor = _SecurityVisitor()
+    visitor.visit(tree)
+    if visitor.violations:
+        lines = "\n".join(f"- {v}" for v in visitor.violations)
+        return Result(preview=Preview(feedback=f"Unsafe code detected:\n{lines}"))
+
+    return Result(preview=Preview(feedback="Valid Python syntax."))

evaluation_function/preview_test.py

@@ -2,28 +2,48 @@
 
 from .preview import Params, preview_function
 
+
 class TestPreviewFunction(unittest.TestCase):
-    """
-    TestCase Class used to test the algorithm.
-    ---
-    Tests are used here to check that the algorithm written
-    is working as it should.
-
-    It's best practice to write these tests first to get a
-    kind of 'specification' for how your algorithm should
-    work, and you should run these tests before committing
-    your code to AWS.
-
-    Read the docs on how to use unittest here:
-    https://docs.python.org/3/library/unittest.html
-
-    Use preview_function() to check your algorithm works
-    as it should.
-    """
-
-    def test_preview(self):
-        response, params = "A", Params()
+
+    def test_valid_python(self):
+        response, params = "x = 1 + 2", Params()
+        result = preview_function(response, params)
+
+        self.assertIn("preview", result)
+        self.assertNotIn("SyntaxError", result["preview"].get("feedback", ""))
+        self.assertNotIn("Unsafe", result["preview"].get("feedback", ""))
+
+    def test_invalid_python(self):
+        response, params = "def foo(:", Params()
+        result = preview_function(response, params)
+
+        self.assertIn("preview", result)
+        self.assertIn("SyntaxError", result["preview"].get("feedback", ""))
+
+    def test_dangerous_import(self):
+        response, params = "import os", Params()
+        result = preview_function(response, params)
+
+        self.assertIn("preview", result)
+        self.assertIn("Unsafe", result["preview"].get("feedback", ""))
+
+    def test_dangerous_from_import(self):
+        response, params = "from subprocess import call", Params()
+        result = preview_function(response, params)
+
+        self.assertIn("preview", result)
+        self.assertIn("Unsafe", result["preview"].get("feedback", ""))
+
+    def test_dangerous_builtin_call(self):
+        response, params = "exec('x=1')", Params()
+        result = preview_function(response, params)
+
+        self.assertIn("preview", result)
+        self.assertIn("Unsafe", result["preview"].get("feedback", ""))
+
+    def test_dunder_access(self):
+        response, params = "x.__class__.__bases__", Params()
         result = preview_function(response, params)
 
         self.assertIn("preview", result)
-        self.assertIsNotNone(result["preview"])
+        self.assertIn("Unsafe", result["preview"].get("feedback", ""))