Yieldability overhaul

.github/workflows/build.yml

@@ -30,7 +30,7 @@ jobs:
             addrsize: "64"
           - os: {name: ubuntu, version: ubuntu-latest}
             addrsize: "64"
-            cflags: "-DHARDSTACKTESTS=1"
+            cflags: "-DHARDSTACKTESTS=2"
           - os: {name: ubuntu-32bit, version: ubuntu-latest}
             addrsize: "32"
             cflags: "-msse3 -mfpmath=sse -D_TIME_BITS=64 -D_FILE_OFFSET_BITS=64"
@@ -62,6 +62,11 @@ jobs:
       run: |
         LDFLAGS=-m${{matrix.addrsize}} CXXFLAGS="-m${{matrix.addrsize}} ${{matrix.cflags}}" CFLAGS="-m${{matrix.addrsize}} ${{matrix.cflags}}" make -j4 config=sanitize ${{ matrix.addrsize == '64' && 'native=1' || ''}} slua-tests
     - name: run tests
+      env:
+        # 32-bit ASAN has a much more limited memory space. If the quarantine zone eats up tons of memory,
+        # we don't have much to work with for our actual tests. Reduce the quarantine zone size
+        # for 32-bit specifically.
+        ASAN_OPTIONS: ${{ matrix.os.name == 'ubuntu-32bit' && 'quarantine_size_mb=64' || '' }}
       run: |
         ./slua-tests
         ./slua-tests --fflags=true
@@ -76,7 +81,7 @@ jobs:
         ./slua-tests -ts=Conformance --codegen -O2 --fflags=true
     - name: make cli
       run: |
-         LDFLAGS=-m${{matrix.addrsize}} CXXFLAGS="-m${{matrix.addrsize}} ${{matrix.cflags}} -DHARDSTACKTESTS=1" CFLAGS="-m${{matrix.addrsize}} ${{matrix.cflags}}" make -j4 config=sanitize ${{ matrix.addrsize == '64' && 'native=1' || ''}} werror=1 slua slua-analyze slua-compile # match config with tests to improve build time
+         LDFLAGS=-m${{matrix.addrsize}} CXXFLAGS="-m${{matrix.addrsize}} ${{matrix.cflags}}" CFLAGS="-m${{matrix.addrsize}} ${{matrix.cflags}}" make -j4 config=sanitize ${{ matrix.addrsize == '64' && 'native=1' || ''}} werror=1 slua slua-analyze slua-compile # match config with tests to improve build time
          ./slua tests/conformance/assert.luau
          ./slua-analyze tests/conformance/assert.luau
          ./slua-compile tests/conformance/assert.luau

.gitignore

@@ -10,6 +10,7 @@
 /crash-*
 /default.prof*
 /fuzz-*
+/fuzz/corpus
 /luau
 /luau-tests
 /luau-analyze

ARES.bt

@@ -30,6 +30,8 @@ const uchar UTAG_UUID = 26;
 const uchar UTAG_DETECTED_EVENT = 28;
 const uchar UTAG_LLEVENTS = 29;
 const uchar UTAG_LLTIMERS = 30;
+const uchar UTAG_STRBUF = 31;
+const uchar UTAG_OPAQUE_BUFFER = 32;
 
 #define LUA_TNONE		(-1)
 
@@ -693,6 +695,14 @@ typedef struct {
         Object tick_wrapper;
         break;
     }
+    case UTAG_STRBUF:
+    {
+        size_t capacity;
+        size_t used;
+        uchar data[used];
+        break;
+    }
+    case UTAG_OPAQUE_BUFFER:
     default:
     {
         size_t length;      /* Size of the data */

CMakeLists.txt

@@ -136,7 +136,7 @@ target_include_directories(Luau.CodeGen PUBLIC CodeGen/include)
 target_link_libraries(Luau.CodeGen PRIVATE Luau.VM Luau.VM.Internals) # Code generation needs VM internals
 target_link_libraries(Luau.CodeGen PUBLIC Luau.Common)
 
-target_compile_features(Luau.VM PRIVATE cxx_std_11)
+target_compile_features(Luau.VM PRIVATE cxx_std_17)
 target_include_directories(Luau.VM PUBLIC VM/include "${PACKAGE_INCLUDE_DIR}")
 target_link_libraries(Luau.VM PUBLIC Luau.Common)
 

Makefile

@@ -111,7 +111,7 @@ ifeq ($(config),release)
 endif
 
 ifeq ($(config),coverage)
-	CXXFLAGS+=-fprofile-instr-generate -fcoverage-mapping
+	CXXFLAGS+=-fprofile-instr-generate -fcoverage-mapping -DLUAU_COVERAGE=1
 	LDFLAGS+=-fprofile-instr-generate
 endif
 
@@ -159,9 +159,9 @@ $(COMPILER_OBJECTS): CXXFLAGS+=-std=c++17 -ICompiler/include -ICommon/include -I
 $(CONFIG_OBJECTS): CXXFLAGS+=-std=c++17 -IConfig/include -ICommon/include -IAst/include -ICompiler/include  -IVM/include
 $(ANALYSIS_OBJECTS): CXXFLAGS+=-std=c++17 -ICommon/include -IAst/include -IAnalysis/include -IConfig/include -ICompiler/include -IVM/include
 $(CODEGEN_OBJECTS): CXXFLAGS+=-std=c++17 -ICommon/include -ICodeGen/include -IVM/include -IVM/src # Code generation needs VM internals
-$(VM_OBJECTS): CXXFLAGS+=-std=c++11 -ICommon/include -IVM/include -Istage/packages/include -I VM/cjson
-$(APR_OBJECTS): CXXFLAGS+=-std=c++11 -Wno-unused-function -Wno-char-subscripts -IVM/include -ICommon/include
-$(CJSON_OBJECTS): CXXFLAGS+=-std=c++11 -Wno-unused-function -Wno-char-subscripts -IVM/include -ICommon/include -I VM/cjson
+$(VM_OBJECTS): CXXFLAGS+=-std=c++17 -ICommon/include -IVM/include -Istage/packages/include -I VM/cjson
+$(APR_OBJECTS): CXXFLAGS+=-std=c++17 -Wno-unused-function -Wno-char-subscripts -IVM/include -ICommon/include
+$(CJSON_OBJECTS): CXXFLAGS+=-std=c++17 -Wno-unused-function -Wno-char-subscripts -IVM/include -ICommon/include -I VM/cjson
 $(REQUIRE_OBJECTS): CXXFLAGS+=-std=c++17 -ICommon/include -IVM/include -IAst/include -IConfig/include -IRequire/include
 $(ISOCLINE_OBJECTS): CXXFLAGS+=-Wno-unused-function -Iextern/isocline/include
 $(TESTS_OBJECTS): CXXFLAGS+=-std=c++17 -ICommon/include -IAst/include -ICompiler/include -IConfig/include -IAnalysis/include -ICodeGen/include -IVM/include -IRequire/include -ICLI/include -Iextern -Istage/packages/include -DDOCTEST_CONFIG_DOUBLE_STRINGIFY -I$(BUILD)

Sources.cmake

@@ -425,15 +425,17 @@ target_sources(Luau.VM PRIVATE
     VM/src/lllevents.h
     VM/src/llltimers.cpp
     VM/src/llltimers.h
+    VM/src/lyieldable.cpp
+    VM/src/lstrbuf.cpp
+    VM/src/lyieldstrlib.h
+    VM/src/lyieldstrlib.cpp
 
     # ServerLua: CJson
     VM/src/cjson/dtoa_config.h
     VM/src/cjson/fpconv.cpp
     VM/src/cjson/fpconv.h
     VM/src/cjson/lua_cjson.cpp
     VM/src/cjson/dtoa_config.h
-    VM/src/cjson/strbuf.cpp
-    VM/src/cjson/strbuf.h
 
     # ServerLua: base64-related routines from APR
     VM/src/apr/apr_base64.cpp

VM/include/llsl.h

@@ -30,6 +30,8 @@ enum class YieldableStatus : uint8_t {
 #define UTAG_DETECTED_EVENT 28
 #define UTAG_LLEVENTS 29
 #define UTAG_LLTIMERS 30
+#define UTAG_STRBUF 31
+#define UTAG_OPAQUE_BUFFER 32
 
 // Internal global names for event/timer managers (hidden from user code)
 #define LLEVENTS_GLOBAL_NAME "/$ LLEvents"

VM/include/lstrbuf.h

@@ -0,0 +1,145 @@
+// ServerLua: Yield-safe string buffer backed by Luau's memory allocator.
+// Based on strbuf.c/h from cjson
+//
+// Lives inside a UTAG_STRBUF tagged userdata for GC-rooted lifetime.
+// Allocating operations require lua_State* for luaM_realloc_ tracking.
+// Non-allocating operations (unsafe appends, length queries) do not.
+#pragma once
+
+#include "lua.h"
+#include "lualib.h"
+#include "llsl.h"
+
+#include <cstddef>
+#include <cstring>
+
+struct lua_YieldSafeStrBuf
+{
+    char* buf;
+    uint32_t size;
+    uint32_t length;
+};
+
+#ifndef STRBUF_DEFAULT_SIZE
+#define STRBUF_DEFAULT_SIZE 128
+#endif
+
+// --- Allocating functions (require lua_State*) ---
+
+LUAI_FUNC void luaYB_init(lua_State* L, lua_YieldSafeStrBuf* s, size_t len);
+LUAI_FUNC void luaYB_free(lua_State* L, lua_YieldSafeStrBuf* s);
+LUAI_FUNC void luaYB_resize(lua_State* L, lua_YieldSafeStrBuf* s, size_t len);
+LUAI_FUNC void luaYB_appendstr(lua_State* L, lua_YieldSafeStrBuf* s, const char* str);
+
+// Replace the strbuf userdata at stack index `idx` with its Lua string equivalent.
+// The strbuf and the resulting string are never on the stack simultaneously.
+// If free_storage is true, the buffer's tracked heap allocation is freed first,
+// reducing peak tracked memory to max(strbuf, string) instead of strbuf + string.
+LUAI_FUNC void luaYB_tostring(lua_State* L, int idx, bool free_storage);
+
+// --- Non-allocating inline functions ---
+
+static inline void luaYB_reset(lua_YieldSafeStrBuf* s)
+{
+    s->length = 0;
+}
+
+static inline int luaYB_allocated(lua_YieldSafeStrBuf* s)
+{
+    return s->buf != NULL;
+}
+
+// Returns bytes remaining, reserving space for a NULL terminator.
+static inline size_t luaYB_space(lua_YieldSafeStrBuf* s)
+{
+    return s->size - s->length - 1;
+}
+
+static inline void luaYB_ensure(lua_State* L, lua_YieldSafeStrBuf* s, size_t len)
+{
+    if (len > luaYB_space(s))
+        luaYB_resize(L, s, s->length + len);
+}
+
+static inline char* luaYB_ptr(lua_YieldSafeStrBuf* s)
+{
+    return s->buf + s->length;
+}
+
+static inline void luaYB_setlen(lua_YieldSafeStrBuf* s, int len)
+{
+    s->length = len;
+}
+
+static inline void luaYB_extend(lua_YieldSafeStrBuf* s, size_t len)
+{
+    s->length += len;
+}
+
+static inline size_t luaYB_len(lua_YieldSafeStrBuf* s)
+{
+    return s->length;
+}
+
+static inline void luaYB_appendchar(lua_State* L, lua_YieldSafeStrBuf* s, const char c)
+{
+    luaYB_ensure(L, s, 1);
+    s->buf[s->length++] = c;
+}
+
+static inline void luaYB_appendchar_unsafe(lua_YieldSafeStrBuf* s, const char c)
+{
+    s->buf[s->length++] = c;
+}
+
+static inline void luaYB_appendmem(lua_State* L, lua_YieldSafeStrBuf* s, const char* c, size_t len)
+{
+    luaYB_ensure(L, s, len);
+    memcpy(s->buf + s->length, c, len);
+    s->length += len;
+}
+
+static inline void luaYB_appendmem_unsafe(lua_YieldSafeStrBuf* s, const char* c, size_t len)
+{
+    memcpy(s->buf + s->length, c, len);
+    s->length += len;
+}
+
+static inline void luaYB_ensurenull(lua_YieldSafeStrBuf* s)
+{
+    s->buf[s->length] = 0;
+}
+
+static inline char* luaYB_data(lua_YieldSafeStrBuf* s, size_t* len)
+{
+    if (len)
+        *len = s->length;
+
+    return s->buf;
+}
+
+// Push buffer contents as an immutable Lua string.
+static inline void luaYB_pushresult(lua_State* L, lua_YieldSafeStrBuf* s)
+{
+    lua_pushlstring(L, s->buf, s->length);
+}
+
+// Pop a string from the top of the stack and append it.
+// Raises a type error if the top value is not string-coercible.
+static inline void luaYB_addvalue(lua_State* L, lua_YieldSafeStrBuf* s)
+{
+    size_t len;
+    const char* str = lua_tolstring(L, -1, &len);
+    if (!str)
+        luaL_typeerror(L, -1, "string");
+    luaYB_appendmem(L, s, str, len);
+    lua_pop(L, 1);
+}
+
+// --- Userdata lifecycle ---
+
+// Registers the UTAG_STRBUF GC destructor. Call once during VM setup.
+LUAI_FUNC void luaYB_setup(lua_State* L);
+
+// Pushes a new lua_YieldSafeStrBuf userdata onto the stack and returns a pointer to it.
+LUAI_FUNC lua_YieldSafeStrBuf* luaYB_push(lua_State* L);

VM/include/lua.h

@@ -395,6 +395,8 @@ LUA_API void lua_concat(lua_State* L, int n);
 LUA_API uintptr_t lua_encodepointer(lua_State* L, uintptr_t p);
 
 LUA_API double lua_clock();
+// ServerLua: per-thread CPU time in seconds (falls back to lua_clock on unsupported platforms)
+LUA_API double lua_cputime();
 
 LUA_API void lua_setrandomseed(lua_State* L, uint64_t seed);
 
@@ -553,6 +555,9 @@ static void populateperms(lua_State *L, bool forUnpersist)
 #if defined(eris_c) || defined(lstrlib_c)
     eris_persist_static(strlib, gmatch_aux)
 #endif
+#if defined(eris_c) || defined(lyieldstrlib_c)
+    eris_persist_cont(lyieldstrlib, yieldable_gmatch_aux_v0, yieldable_gmatch_aux_v0_k)
+#endif
 #if defined(eris_c) || defined(lutf8lib_c)
     eris_persist_static(utf8lib, iter_aux)
 #endif
@@ -567,12 +572,12 @@ static void populateperms(lua_State *L, bool forUnpersist)
     eris_persist_static_cont(corolib, auxwrapy, auxwrapcont)
 #endif
 #if defined(eris_c) || defined(lllevents_c)
-    eris_persist_static_cont(llevents, llevents_handle_event_init, llevents_handle_event_cont)
+    eris_persist_static_cont(llevents, llevents_handle_event_v0, llevents_handle_event_v0_k)
     eris_persist_static_cont(llevents, llevents_once_wrapper, llevents_once_wrapper_cont)
     eris_persist_static(llevents, timer_wrapper_guard)
 #endif
 #if defined(eris_c) || defined(llltimers_c)
-    eris_persist_static_cont(llltimers, lltimers_tick_init, lltimers_tick_cont)
+    eris_persist_static_cont(llltimers, lltimers_tick_v0, lltimers_tick_v0_k)
     eris_persist_cont(llltimers, timer_event_wrapper, timer_event_wrapper_cont)
 #endif
 #if defined(eris_c)

VM/include/luaconf.h

@@ -3,8 +3,10 @@
 #pragma once
 
 // When debugging complex issues, consider enabling one of these:
-// This will reallocate the stack very aggressively at every opportunity; use this with asan to catch stale stack pointers
+// This will reallocate the stack very aggressively at most opportunities; use this with asan to catch stale stack pointers
 // #define HARDSTACKTESTS 1
+// This will reallocate the stack very aggressively at every opportunity, including whenever a metamethod could have been invoked
+// #define HARDSTACKTESTS 2
 // This will call GC validation very aggressively at every incremental GC step; use this with caution as it's SLOW
 // #define HARDMEMTESTS 1
 // This will call GC validation very aggressively at every GC opportunity; use this with caution as it's VERY SLOW

VM/include/lualib.h

@@ -128,6 +128,8 @@ LUALIB_API int luaopen_os(lua_State* L);
 
 #define LUA_STRLIBNAME "string"
 LUALIB_API int luaopen_string(lua_State* L);
+// ServerLua: Registers base (non-yieldable) pattern-matching functions on table at stack top.
+LUALIB_API void luaopen_string_base(lua_State* L);
 
 #define LUA_BITLIBNAME "bit32"
 LUALIB_API int luaopen_bit32(lua_State* L);