diff --git a/fix_sft_audio_mode_check.py b/fix_sft_audio_mode_check.py new file mode 100644 index 0000000..b60e093 --- /dev/null +++ b/fix_sft_audio_mode_check.py @@ -0,0 +1,42 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Fix the missing mode check in appendDotTrialTimeline + +on_stim_start = """ postResponseContent: config.rtTask.enabled ? config.rtTask.postResponseContent : "stimulus", + onStimulusPhaseStart: () => { + const trial = trialProvider(); + if (!trial || !audioService || config.audio.mode !== "audiovisual") return; + let targetSalience = 0; + const code = trial.stimCode; + if (code.length === 2) { + const audioChar = code[1]; // Assume channel 2 is audio + if (audioChar === 'H') targetSalience = trial.salience.high; + else if (audioChar === 'L') targetSalience = trial.salience.low; + else if (audioChar === 'x') targetSalience = 0; + else targetSalience = trial.salience.high; // fallback + } else { + targetSalience = trial.salience.high; + } + + if (targetSalience > 0) { + const baseVol = config.audio.volume; + const vol = baseVol * targetSalience; + audioService.playTone(config.audio.frequencyHz, config.audio.durationMs, { + waveform: config.audio.waveform, + volume: vol + }); + } + }, + onResponse: (response: { key: string | null; rtMs: number | null }, data: Record) => {""" + +content = re.sub(r' postResponseContent: config\.rtTask\.enabled \? config\.rtTask\.postResponseContent : "stimulus",\n onStimulusPhaseStart: \(\) => \{\n const trial = trialProvider\(\);\n if \(!trial \|\| !audioService\) return;\n let targetSalience = 0;\n const code = trial\.stimCode;\n if \(code\.length === 2\) \{\n const audioChar = code\[1\]; // Assume channel 2 is audio\n if \(audioChar === \'H\'\) targetSalience = trial\.salience\.high;\n else if \(audioChar === \'L\'\) targetSalience = trial\.salience\.low;\n else if \(audioChar === \'x\'\) targetSalience = 0;\n else targetSalience = trial\.salience\.high; // fallback\n \} else \{\n targetSalience = trial\.salience\.high;\n \}\n \n if \(targetSalience > 0\) \{\n const baseVol = config\.audio\.volume;\n const vol = baseVol \* targetSalience;\n audioService\.playTone\(config\.audio\.frequencyHz, config\.audio\.durationMs, \{\n waveform: config\.audio\.waveform,\n volume: vol\n \}\);\n \}\n \},\n onResponse: \(response: \{ key: string \| null; rtMs: number \| null \}, data: Record\) => \{', on_stim_start, content) + + +with open(file_path, "w") as f: + f.write(content) + +print("Fixed audio mode check") diff --git a/fix_sft_config.py b/fix_sft_config.py new file mode 100644 index 0000000..987cc37 --- /dev/null +++ b/fix_sft_config.py @@ -0,0 +1,41 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Fix interface: find the end of interface SftParsedConfig and insert audio before it +# Revert previous replacement first +content = re.sub(r' audio: \{\n waveform: OscillatorType;\n frequencyHz: number;\n durationMs: number;\n volume: number;\n \};\n', '', content) + +# Correctly insert audio into SftParsedConfig +sft_parsed_config_re = r'(interface SftParsedConfig \{[^}]+\n display: \{[^}]+\};)' +audio_interface = """ + audio: { + waveform: OscillatorType; + frequencyHz: number; + durationMs: number; + volume: number; + };""" +content = re.sub(sft_parsed_config_re, r'\1' + audio_interface, content) + +# Fix parseSftConfig audio parsing placement +content = re.sub(r'\n const stimulusAudioRaw = asObject\(stimulusRaw\?\.audio\);\n const audio = \{\n waveform: \(asString\(stimulusAudioRaw\?\.waveform\) \|\| "sine"\) as OscillatorType,\n frequencyHz: toPositiveNumber\(stimulusAudioRaw\?\.frequencyHz \?\? stimulusAudioRaw\?\.frequency_hz, 440\),\n durationMs: toPositiveNumber\(stimulusAudioRaw\?\.durationMs \?\? stimulusAudioRaw\?\.duration_ms, legacyTiming\.stimulusMs\),\n volume: toUnitNumber\(stimulusAudioRaw\?\.volume, 0\.25\),\n \};\n', '', content) + +audio_parsing = """ + const stimulusAudioRaw = asObject(stimulusRaw?.audio); + const audio = { + waveform: (asString(stimulusAudioRaw?.waveform) || "sine") as OscillatorType, + frequencyHz: toPositiveNumber(stimulusAudioRaw?.frequencyHz ?? stimulusAudioRaw?.frequency_hz, 440), + durationMs: toPositiveNumber(stimulusAudioRaw?.durationMs ?? stimulusAudioRaw?.duration_ms, legacyTiming.stimulusMs), + volume: toUnitNumber(stimulusAudioRaw?.volume, 0.25), + }; +""" + +content = re.sub(r'(const legacyTiming = \{[^}]+\};)', r'\1' + audio_parsing, content) + + +with open(file_path, "w") as f: + f.write(content) + +print("Fixed SFT Config") diff --git a/fix_sft_config2.py b/fix_sft_config2.py new file mode 100644 index 0000000..7059da3 --- /dev/null +++ b/fix_sft_config2.py @@ -0,0 +1,31 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Add audio to SftParsedConfig +audio_interface = """ + audio: { + waveform: OscillatorType; + frequencyHz: number; + durationMs: number; + volume: number; + };""" + +sft_parsed_config_re = r'(interface SftParsedConfig \{.*?\ndisplay: \{.*?\};)' + +# Instead of using regex for adding the interface which seems tricky, let's just use string replace. +lines = content.split('\n') +for i, line in enumerate(lines): + if " display: {" in line and "SftParsedConfig" in "\n".join(lines[max(0, i-20):i]): + # Check if we already have it + if "audio: {" not in "\n".join(lines[i:i+20]): + lines.insert(i, " audio: {\n waveform: OscillatorType;\n frequencyHz: number;\n durationMs: number;\n volume: number;\n };") + break + +content = "\n".join(lines) +with open(file_path, "w") as f: + f.write(content) + +print("Fixed SFT Config 2") diff --git a/fix_sft_timeline.py b/fix_sft_timeline.py new file mode 100644 index 0000000..008cf97 --- /dev/null +++ b/fix_sft_timeline.py @@ -0,0 +1,50 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# I messed up the regex substitution for appendDotTrialTimeline. +# It added the onStimulusPhaseStart inside the argument object of appendDotTrialTimeline instead of buildJsPsychRtTimelineNodes. +# Let's clean it up and add it correctly. + +# The bad part was inserted at the end of the appendDotTrialTimeline object in runTrial. Let's revert and do it right. + +content = re.sub(r' onStimulusPhaseStart: \(\) => \{\n const trial = trialProvider\(\);\n.*?\n \},\n onResponse:', ' onResponse:', content, flags=re.DOTALL) + +# Let's manually replace it inside buildJsPsychRtTimelineNodes + +on_stim_start = """ postResponseContent: config.rtTask.enabled ? config.rtTask.postResponseContent : "stimulus", + onStimulusPhaseStart: () => { + const trial = trialProvider(); + if (!trial || !audioService) return; + let targetSalience = 0; + const code = trial.stimCode; + if (code.length === 2) { + const audioChar = code[1]; // Assume channel 2 is audio + if (audioChar === 'H') targetSalience = trial.salience.high; + else if (audioChar === 'L') targetSalience = trial.salience.low; + else if (audioChar === 'x') targetSalience = 0; + else targetSalience = trial.salience.high; // fallback + } else { + targetSalience = trial.salience.high; + } + + if (targetSalience > 0) { + const baseVol = config.audio.volume; + const vol = baseVol * targetSalience; + audioService.playTone(config.audio.frequencyHz, config.audio.durationMs, { + waveform: config.audio.waveform, + volume: vol + }); + } + }, + onResponse: (response: { key: string | null; rtMs: number | null }, data: Record) => {""" + +content = re.sub(r' postResponseContent: config.rtTask.enabled \? config.rtTask.postResponseContent : "stimulus",\n onResponse: \(response: \{ key: string \| null; rtMs: number \| null \}, data: Record\) => \{', on_stim_start, content) + + +with open(file_path, "w") as f: + f.write(content) + +print("Fixed SFT Timeline") diff --git a/packages/core/src/engines/jspsychRtTask.ts b/packages/core/src/engines/jspsychRtTask.ts index dbe84cf..dce0c27 100644 --- a/packages/core/src/engines/jspsychRtTask.ts +++ b/packages/core/src/engines/jspsychRtTask.ts @@ -33,6 +33,7 @@ export interface JsPsychRtTimelineConfig { }; postResponseContent?: "blank" | "stimulus"; onResponse?: (response: { key: string | null; rtMs: number | null }, data: Record) => void; + onStimulusPhaseStart?: () => void; } export function initStandardJsPsych(args: { @@ -69,6 +70,7 @@ export function buildJsPsychRtTimelineNodes(config: JsPsychRtTimelineConfig): an feedback, postResponseContent = "stimulus", onResponse, + onStimulusPhaseStart, } = config; const timeline: any[] = []; @@ -155,6 +157,11 @@ export function buildJsPsychRtTimelineNodes(config: JsPsychRtTimelineConfig): an response_ends_trial: responseTerminatesTrial, trial_duration: Math.max(0, Math.round(segment.durationMs)), data: { ...baseData, phase: segment.phase }, + on_start: () => { + if (segment.showStimulus && onStimulusPhaseStart && !responseSeen) { + onStimulusPhaseStart(); + } + }, on_finish: (data: Record) => { if (!responseSeen) { const response = extractJsPsychTrialResponse(data); diff --git a/patch_dots_audio.py b/patch_dots_audio.py new file mode 100644 index 0000000..78a1f00 --- /dev/null +++ b/patch_dots_audio.py @@ -0,0 +1,16 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Let's see if we can add logic to avoid drawing the auditory dot if it's meant to be an auditory channel. +# Wait, SFT is a redundancy task. Does it have to be visual? +# "a valid sft config could be a redundant target task with one auditory stimulus and one visual stimulus... just like the two visual stimuli." +# So if it's auditory, maybe channel 2 shouldn't be drawn at all! +# We can determine if the task has an auditory channel by checking `config.audio.enabled`? We didn't add `enabled`. Let's just say if it's "audiovisual" mode. +# Or we can just add `audio: { enabled: boolean; ... }` to `SftParsedConfig`. +# Actually, the user asked to "Ensure the stimulus can be used with the staircase module and that it can be integrated seamlessly such that a valid sft config could be a redundant target task with one auditory stimulus and one visual stimulus (that can be presented together or in isolation...)" +# Let's add `mode: "visual" | "audiovisual"` to the display config or just infer it if audio config is present. +# It might be simpler: if `stimulus.audio.enabled` is true, then channel "B" (index 1) is the auditory stimulus. +# Or we can add a config flag: `config.stimulus.mode = "audiovisual"`. diff --git a/patch_jspsych_rt.py b/patch_jspsych_rt.py new file mode 100644 index 0000000..1145c0c --- /dev/null +++ b/patch_jspsych_rt.py @@ -0,0 +1,37 @@ +import re + +file_path = "packages/core/src/engines/jspsychRtTask.ts" +with open(file_path, "r") as f: + content = f.read() + +# Add onStimulusPhaseStart to JsPsychRtTimelineConfig +config_addition = """ onResponse?: (response: { key: string | null; rtMs: number | null }, data: Record) => void; + onStimulusPhaseStart?: () => void; +""" +content = re.sub(r' onResponse\?: \(response: \{ key: string \| null; rtMs: number \| null \}, data: Record\) => void;\n', config_addition, content) + +# Add onStimulusPhaseStart extraction +destructure_addition = """ postResponseContent = "stimulus", + onResponse, + onStimulusPhaseStart, +""" +content = re.sub(r' postResponseContent = "stimulus",\n onResponse,\n', destructure_addition, content) + + +# Add on_start callback to response segments +on_start_addition = """ + data: { ...baseData, phase: segment.phase }, + on_start: () => { + if (segment.showStimulus && onStimulusPhaseStart && !responseSeen) { + onStimulusPhaseStart(); + } + }, + on_finish: (data: Record) => {""" + +content = re.sub(r'\n data: \{ \.\.\.baseData, phase: segment\.phase \},\n on_finish: \(data: Record\) => \{', on_start_addition, content) + + +with open(file_path, "w") as f: + f.write(content) + +print("Patched jsPsychRtTask successfully") diff --git a/patch_sft_audio_mode.py b/patch_sft_audio_mode.py new file mode 100644 index 0000000..4924baa --- /dev/null +++ b/patch_sft_audio_mode.py @@ -0,0 +1,44 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Add mode to SftParsedConfig +audio_interface = """ + audio: { + mode: "visual" | "audiovisual"; + waveform: OscillatorType; +""" +content = re.sub(r'\n audio: \{\n waveform: OscillatorType;', audio_interface, content) + + +# Add mode parsing +audio_parsing = """ + const stimulusAudioRaw = asObject(stimulusRaw?.audio); + const audioMode = (asString(stimulusAudioRaw?.mode) || "visual").toLowerCase() === "audiovisual" ? "audiovisual" : "visual"; + const audio = { + mode: audioMode as "visual" | "audiovisual", + waveform: (asString(stimulusAudioRaw?.waveform) || "sine") as OscillatorType,""" +content = re.sub(r'\n const stimulusAudioRaw = asObject\(stimulusRaw\?\.audio\);\n const audio = \{\n waveform: \(asString\(stimulusAudioRaw\?\.waveform\) \|\| "sine"\) as OscillatorType,', audio_parsing, content) + + +# Update dotsFromStimCode to not draw the second dot if audiovisual +# First, pass config to dotsFromStimCode or just use mode +content = re.sub(r'const dots = dotsFromStimCode\(trial\.stimCode, trial\.salience\);', r'const dots = dotsFromStimCode(trial.stimCode, trial.salience, config.audio.mode);', content) + +# Then update dotsFromStimCode function +dots_func = """function dotsFromStimCode(stimCode: string, salience: { high: number; low: number }, mode: "visual" | "audiovisual"): Array<{ loc: "A" | "B"; luminance: number }> { + const [a, b] = normalizeStimCode(stimCode).split(""); + const dots: Array<{ loc: "A" | "B"; luminance: number }> = []; + if (a !== "x") dots.push({ loc: "A", luminance: a === "H" ? salience.high : salience.low }); + if (mode === "visual" && b !== "x") dots.push({ loc: "B", luminance: b === "H" ? salience.high : salience.low }); + return dots; +}""" +content = re.sub(r'function dotsFromStimCode\(stimCode: string, salience: \{ high: number; low: number \}\): Array<\{ loc: "A" \| "B"; luminance: number \}> \{\n const \[a, b\] = normalizeStimCode\(stimCode\)\.split\(""\);\n const dots: Array<\{ loc: "A" \| "B"; luminance: number \}> = \[\];\n if \(a !== "x"\) dots\.push\(\{ loc: "A", luminance: a === "H" \? salience\.high : salience\.low \}\);\n if \(b !== "x"\) dots\.push\(\{ loc: "B", luminance: b === "H" \? salience\.high : salience\.low \}\);\n return dots;\n\}', dots_func, content) + + +with open(file_path, "w") as f: + f.write(content) + +print("Patched audio mode") diff --git a/patch_sft_config.py b/patch_sft_config.py new file mode 100644 index 0000000..e8042c6 --- /dev/null +++ b/patch_sft_config.py @@ -0,0 +1,37 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# 1. Update SftParsedConfig interface +interface_addition = """ audio: { + waveform: OscillatorType; + frequencyHz: number; + durationMs: number; + volume: number; + }; +""" +content = re.sub(r'(interface SftParsedConfig \{[^\}]+display: \{[^\}]+\};)', r'\1\n' + interface_addition, content) + +# 2. Update parseSftConfig function +audio_parsing_addition = """ + const stimulusAudioRaw = asObject(stimulusRaw?.audio); + const audio = { + waveform: (asString(stimulusAudioRaw?.waveform) || "sine") as OscillatorType, + frequencyHz: toPositiveNumber(stimulusAudioRaw?.frequencyHz ?? stimulusAudioRaw?.frequency_hz, 440), + durationMs: toPositiveNumber(stimulusAudioRaw?.durationMs ?? stimulusAudioRaw?.duration_ms, legacyTiming.stimulusMs), + volume: toUnitNumber(stimulusAudioRaw?.volume, 0.25), + }; +""" +content = re.sub(r'(const legacyTiming = \{)', audio_parsing_addition + r'\n \1', content) + +# 3. Add to return object of parseSftConfig +return_addition = """ audio, +""" +content = re.sub(r'(display: \{[^\}]+\},)', r'\1\n' + return_addition, content) + +with open(file_path, "w") as f: + f.write(content) + +print("Patched SFT Config successfully") diff --git a/patch_sft_runner.py b/patch_sft_runner.py new file mode 100644 index 0000000..0c0a3fd --- /dev/null +++ b/patch_sft_runner.py @@ -0,0 +1,69 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Instantiate AudioService in runSftTask +audio_service_instantiation = """ + let jsPsych: ReturnType | null = null; + const audioService = new AudioService(); + const orchestrator = new TaskOrchestrator(context); +""" +content = re.sub(r'\n let jsPsych: ReturnType \| null = null;\n const orchestrator = new TaskOrchestrator\(context\);\n', audio_service_instantiation, content) + +# Pass audioService to appendStaircaseTimeline +staircase_audio = """ timeline: staircaseTimeline, + container: root, + config: parsed, + audioService, +""" +content = re.sub(r' timeline: staircaseTimeline,\n container: root,\n config: parsed,\n', staircase_audio, content) + +# Update appendStaircaseTimeline args +append_staircase_args = """function appendStaircaseTimeline(args: { + timeline: any[]; + container: HTMLElement; + config: SftParsedConfig; + audioService?: AudioService; +""" +content = re.sub(r'function appendStaircaseTimeline\(args: \{\n timeline: any\[\];\n container: HTMLElement;\n config: SftParsedConfig;\n', append_staircase_args, content) + +# Destructure audioService in appendStaircaseTimeline +staircase_destructure = """ const { timeline, container, config, audioService, rng, allowedKeys, staircaseRecords, eventLogger } = args;""" +content = re.sub(r' const \{ timeline, container, config, rng, allowedKeys, staircaseRecords, eventLogger \} = args;', staircase_destructure, content) + +# Pass audioService to appendDotTrialTimeline in appendStaircaseTimeline +staircase_dot_audio = """ appendDotTrialTimeline({ + timeline, + config, + audioService, +""" +content = re.sub(r' appendDotTrialTimeline\(\{\n timeline,\n config,\n', staircase_dot_audio, content) + +# Pass audioService to appendDotTrialTimeline in runTrial +run_trial_audio = """ appendDotTrialTimeline({ + timeline: trialTimeline, + config: parsed, + audioService, +""" +content = re.sub(r' appendDotTrialTimeline\(\{\n timeline: trialTimeline,\n config: parsed,\n', run_trial_audio, content) + +# Clean up audioService on task complete +on_dispose = """ }; + }, + renderInstruction: createInstructionRenderer({""" +content = re.sub(r' \};\n \},\n renderInstruction: createInstructionRenderer\(\{', on_dispose, content) + +# Actually, TaskOrchestrator doesn't have an onDispose hook, let's just make sure it disposes when we're done. Wait, TaskAdapter context might have dispose but JS GC will handle it mostly. The AudioService class does have a dispose method, let's add it to a try/finally block around orchestrator.run() or similar. +# Alternatively, we can add it to getTaskMetadata which is called at the end. +get_metadata_addition = """ getTaskMetadata: (sessionResult) => { + audioService.dispose(); +""" +content = re.sub(r' getTaskMetadata: \(sessionResult\) => \{\n', get_metadata_addition, content) + + +with open(file_path, "w") as f: + f.write(content) + +print("Patched SFT Runner") diff --git a/patch_sft_timeline.py b/patch_sft_timeline.py new file mode 100644 index 0000000..616cccd --- /dev/null +++ b/patch_sft_timeline.py @@ -0,0 +1,74 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Add AudioService import +audio_import = 'import { AudioService } from "@experiments/core";\n' +content = audio_import + content + +# Modify appendDotTrialTimeline arguments +append_args = """function appendDotTrialTimeline(args: { + timeline: any[]; + config: SftParsedConfig; + audioService?: AudioService; +""" +content = re.sub(r'function appendDotTrialTimeline\(args: \{\n timeline: any\[\];\n config: SftParsedConfig;', append_args, content) + +# Destructure audioService +destructure_args = """ const { timeline, config, audioService, trialProvider, allowedKeys, rng, phasePrefix, onResponse, dataContext, feedback } = args;""" +content = re.sub(r' const \{ timeline, config, trialProvider, allowedKeys, rng, phasePrefix, onResponse, dataContext, feedback \} = args;', destructure_args, content) + + +# Add onStimulusPhaseStart to buildJsPsychRtTimelineNodes +on_stim_start = """ onStimulusPhaseStart: () => { + const trial = trialProvider(); + if (!trial || !audioService) return; + if (trial.stimCategory === "AB" || trial.stimCategory === "AN" || trial.stimCategory === "NB" || trial.stimCategory === "NN" || trial.stimCode) { + if (trial.showRuleCue === false && trial.stimCode.includes("x")) { + // In SFT visual/audio might be combined. For pure visual it's fine. For pure audio we might need a flag. + // We'll play audio for any auditory components if available, or base it on salience. + // "salience" determines volume. + } + } + // If we have an audio setting and salience for auditory > 0, we can play it. + // SFT has high/low salience. For audio, high salience could mean full volume, low means reduced. + // E.g., if dot salience works for vision, we can use it to scale audio volume. + // Actually, we'll play the tone if the trial implies an auditory stimulus. + // If SFT becomes redundant target (audio + visual), we might need to know if the trial includes audio. + // For now, if audioService is provided and salience > 0, we play. We use trial.salience.high / low. + + let targetSalience = 0; + // If the stimCode indicates high/low salience for the target, use it. + // We can assume standard SFT codes like 'HH', 'Hx', 'xH' etc. + // Usually channel 1 is visual, channel 2 could be auditory? + // For a redundant target task, "Hx" means High visual, no audio. "xH" means no visual, High audio. "HH" means High visual, High audio. + const code = trial.stimCode; + if (code.length === 2) { + const audioChar = code[1]; // Assume channel 2 is audio + if (audioChar === 'H') targetSalience = trial.salience.high; + else if (audioChar === 'L') targetSalience = trial.salience.low; + else if (audioChar === 'x') targetSalience = 0; + else targetSalience = trial.salience.high; // fallback + } else { + targetSalience = trial.salience.high; + } + + if (targetSalience > 0) { + const baseVol = config.audio.volume; + // Scale volume by salience. Salience is between 0 and 1. + const vol = baseVol * targetSalience; + audioService.playTone(config.audio.frequencyHz, config.audio.durationMs, { + waveform: config.audio.waveform, + volume: vol + }); + } + }, + onResponse: """ +content = re.sub(r' onResponse: ', on_stim_start, content) + +with open(file_path, "w") as f: + f.write(content) + +print("Patched SFT Timeline") diff --git a/patch_sft_trial.py b/patch_sft_trial.py new file mode 100644 index 0000000..6c6124d --- /dev/null +++ b/patch_sft_trial.py @@ -0,0 +1,10 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Since `stimCode` is already in PlannedTrial and TrialRecord, and it natively encodes channels (e.g. "HH", "Hx"), +# we just need to make sure the logging logs audio correctly if we want. +# Actually, the `channel1` and `channel2` already extract this in the CSV logger or data pipeline? +# Let's check how channel1 and channel2 are generated. diff --git a/patch_staircase.py b/patch_staircase.py new file mode 100644 index 0000000..1bbca14 --- /dev/null +++ b/patch_staircase.py @@ -0,0 +1,18 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# During staircase, we need to make sure the staircase trial plays the audio if mode="audiovisual" +# Let's check how the staircase trial is set up. + +# In `appendStaircaseTimeline`, the `activeTrial` is set to `stimCode: "Hx"` which implies only channel 1 is active (visual). +# If the task is "audiovisual", maybe the staircase should run on the visual channel, or both? +# The staircase typically estimates threshold luminance and we just use the same multiplier for audio volume? +# The task says: "Ensure the stimulus can be used with the staircase module and that it can be integrated seamlessly such that a valid sft config could be a redundant target task with one auditory stimulus and one visual stimulus (that can be presented together or in isolation, at various salience levels, trial to trial just like the two visual stimuli.)" +# Wait, if mode is "audiovisual", how do we know if staircase is testing visual or auditory? +# If "Hx", it is testing visual. What if the user wants to staircase the auditory stimulus? They would set "xH". +# Wait, the staircase in SFT currently hardcodes `stimCode: "Hx"`. +# We should allow staircase to configure which channel to test, or just randomly pick? +# Usually staircase in SFT tests one channel. Let's make the staircase `stimCode` configurable, defaulting to "Hx" if not provided. diff --git a/patch_staircase2.py b/patch_staircase2.py new file mode 100644 index 0000000..823f519 --- /dev/null +++ b/patch_staircase2.py @@ -0,0 +1,32 @@ +import re + +file_path = "tasks/sft/src/index.ts" +with open(file_path, "r") as f: + content = f.read() + +# Add `stimCode` to StaircaseSpec +staircase_spec_re = r'interface StaircaseSpec \{\n enabled: boolean;\n nTrials: number;' +staircase_spec_addition = """interface StaircaseSpec { + enabled: boolean; + stimCode: string; + nTrials: number;""" +content = re.sub(staircase_spec_re, staircase_spec_addition, content) + +# Update parseStaircase +parse_staircase_re = r' enabled,\n nTrials: toPositiveNumber\(raw\.n_trials \?\? raw\.nTrials, 20\),' +parse_staircase_addition = """ enabled, + stimCode: asString(raw.stim_code ?? raw.stimCode) || "Hx", + nTrials: toPositiveNumber(raw.n_trials ?? raw.nTrials, 20),""" +content = re.sub(parse_staircase_re, parse_staircase_addition, content) + + +# Update appendStaircaseTimeline to use config.staircase.stimCode instead of hardcoded "Hx" +append_staircase_re = r'stimCode: "Hx",' +append_staircase_addition = """stimCode: staircase.stimCode,""" +content = re.sub(append_staircase_re, append_staircase_addition, content) + + +with open(file_path, "w") as f: + f.write(content) + +print("Patched Staircase") diff --git a/tasks/sft/src/index.ts b/tasks/sft/src/index.ts index 3624877..7537eb9 100644 --- a/tasks/sft/src/index.ts +++ b/tasks/sft/src/index.ts @@ -1,3 +1,4 @@ +import { AudioService } from "@experiments/core"; import { QuestBinaryStaircase, buildLinearRange, @@ -95,6 +96,14 @@ interface SftParsedConfig { responseDeadlineMs: number; responseTerminatesTrial: boolean; }; + audio: { + mode: "visual" | "audiovisual"; + waveform: OscillatorType; + + frequencyHz: number; + durationMs: number; + volume: number; + }; display: { aperturePx: number; dotOffsetPx: number; @@ -220,6 +229,7 @@ interface TrialRecord { interface StaircaseSpec { enabled: boolean; + stimCode: string; nTrials: number; stimDbMin: number; stimDbMax: number; @@ -277,6 +287,7 @@ async function runSftTask(context: TaskAdapterContext): Promise { applyGlobalSalience(parsed, parsed.salience); let jsPsych: ReturnType | null = null; + const audioService = new AudioService(); const orchestrator = new TaskOrchestrator(context); applyTaskInstructionConfig(context.taskConfig, { ...parsed.instructions, @@ -453,6 +464,7 @@ async function runSftTask(context: TaskAdapterContext): Promise { timeline: staircaseTimeline, container: root, config: parsed, + audioService, rng, allowedKeys, staircaseRecords, @@ -473,6 +485,7 @@ async function runSftTask(context: TaskAdapterContext): Promise { suffix: "sft_trials", }, getTaskMetadata: (sessionResult) => { + audioService.dispose(); const records = sessionResult.blocks.flatMap((b: any) => b.trialResults) as TrialRecord[]; eventLogger.emit("task_complete", { task: "sft", runner: "jspsych", nTrials: records.length }); return { @@ -494,12 +507,13 @@ function appendStaircaseTimeline(args: { timeline: any[]; container: HTMLElement; config: SftParsedConfig; + audioService?: AudioService; rng: () => number; allowedKeys: string[]; staircaseRecords: StaircaseRecord[]; eventLogger: EventLogger; }): void { - const { timeline, container, config, rng, allowedKeys, staircaseRecords, eventLogger } = args; + const { timeline, container, config, audioService, rng, allowedKeys, staircaseRecords, eventLogger } = args; const staircase = config.staircase; if (!staircase?.enabled) return; @@ -541,7 +555,7 @@ function appendStaircaseTimeline(args: { trialIndex: timelineIndex, rule: "OR", layout: "center", - stimCode: "Hx", + stimCode: staircase.stimCode, stimCategory: "AN", salience: { high: dbToLuminance(stimDb), low: 0 }, showRuleCue: false, @@ -554,6 +568,7 @@ function appendStaircaseTimeline(args: { appendDotTrialTimeline({ timeline, config, + audioService, trialProvider: () => activeTrial?.trial ?? null, allowedKeys, rng, @@ -624,6 +639,8 @@ function appendStaircaseTimeline(args: { function appendDotTrialTimeline(args: { timeline: any[]; config: SftParsedConfig; + audioService?: AudioService; + trialProvider: () => PlannedTrial | null; allowedKeys: string[]; rng: () => number; @@ -637,7 +654,7 @@ function appendDotTrialTimeline(args: { viewProvider: () => { text: string; color: string } | null; }; }): void { - const { timeline, config, trialProvider, allowedKeys, rng, phasePrefix, onResponse, dataContext, feedback } = args; + const { timeline, config, audioService, trialProvider, allowedKeys, rng, phasePrefix, onResponse, dataContext, feedback } = args; const responseTerminatesTrial = config.rtTask.responseTerminatesTrial; const fallbackFixationMs = jitter(config.timing.fixationMs, rng); @@ -721,6 +738,30 @@ function appendDotTrialTimeline(args: { } : undefined, postResponseContent: config.rtTask.enabled ? config.rtTask.postResponseContent : "stimulus", + onStimulusPhaseStart: () => { + const trial = trialProvider(); + if (!trial || !audioService || config.audio.mode !== "audiovisual") return; + let targetSalience = 0; + const code = trial.stimCode; + if (code.length === 2) { + const audioChar = code[1]; // Assume channel 2 is audio + if (audioChar === 'H') targetSalience = trial.salience.high; + else if (audioChar === 'L') targetSalience = trial.salience.low; + else if (audioChar === 'x') targetSalience = 0; + else targetSalience = trial.salience.high; // fallback + } else { + targetSalience = trial.salience.high; + } + + if (targetSalience > 0) { + const baseVol = config.audio.volume; + const vol = baseVol * targetSalience; + audioService.playTone(config.audio.frequencyHz, config.audio.durationMs, { + waveform: config.audio.waveform, + volume: vol + }); + } + }, onResponse: (response: { key: string | null; rtMs: number | null }, data: Record) => { onResponse(response, trialProvider(), data); }, @@ -788,6 +829,7 @@ function parseSftConfig(config: JSONObject, selection: TaskAdapterContext["selec const stimulusRaw = asObject(config.stimulus); const salienceRaw = asObject(stimulusRaw?.salience_levels); const conditionCodes = asArray(stimulusRaw?.condition_codes).map((v) => asString(v)).filter((v): v is string => Boolean(v)); + const legacyTiming = { fixationMs: toNonNegativeNumber(asObject(config.timing)?.fixation_truncexp ? (asObject(asObject(config.timing)?.fixation_truncexp)?.mean ?? 500) : 500, 500), blankMs: toNonNegativeNumber(asObject(config.timing)?.blank_ms, 66), @@ -795,6 +837,16 @@ function parseSftConfig(config: JSONObject, selection: TaskAdapterContext["selec responseDeadlineMs: toPositiveNumber(asObject(config.timing)?.response_deadline_ms, 3000), responseTerminatesTrial: asObject(config.timing)?.response_terminates_trial !== false, }; + const stimulusAudioRaw = asObject(stimulusRaw?.audio); + const audioMode = (asString(stimulusAudioRaw?.mode) || "visual").toLowerCase() === "audiovisual" ? "audiovisual" : "visual"; + const audio = { + mode: audioMode as "visual" | "audiovisual", + waveform: (asString(stimulusAudioRaw?.waveform) || "sine") as OscillatorType, + frequencyHz: toPositiveNumber(stimulusAudioRaw?.frequencyHz ?? stimulusAudioRaw?.frequency_hz, 440), + durationMs: toPositiveNumber(stimulusAudioRaw?.durationMs ?? stimulusAudioRaw?.duration_ms, legacyTiming.stimulusMs), + volume: toUnitNumber(stimulusAudioRaw?.volume, 0.25), + }; + const taskRaw = asObject(config.task); const rtRaw = asObject(taskRaw?.rtTask); @@ -904,6 +956,8 @@ function parseSftConfig(config: JSONObject, selection: TaskAdapterContext["selec canvasBorder: asString(asObject(config.display)?.canvas_border) || "2px solid #444", cueColor: asString(asObject(config.display)?.cue_color) || "#0f172a", }, + audio, + responses, responseSemantics, salience: { @@ -935,6 +989,7 @@ function parseStaircase(config: JSONObject): StaircaseSpec | null { const enabled = Boolean(raw.enabled); return { enabled, + stimCode: asString(raw.stim_code ?? raw.stimCode) || "Hx", nTrials: toPositiveNumber(raw.n_trials ?? raw.nTrials, 20), stimDbMin: toFiniteNumber(raw.stim_db_min, -2.5), stimDbMax: toFiniteNumber(raw.stim_db_max, -0.2), @@ -1125,7 +1180,7 @@ function drawStimulusPhase(canvas: HTMLCanvasElement, config: SftParsedConfig, t }, ({ centerX, centerY }) => { if (!trial) return; const positions = dotPositions(centerX, centerY, trial.layout, config.display.dotOffsetPx); - const dots = dotsFromStimCode(trial.stimCode, trial.salience); + const dots = dotsFromStimCode(trial.stimCode, trial.salience, config.audio.mode); for (const dot of dots) { const p = positions[dot.loc]; ctx.beginPath(); @@ -1177,11 +1232,11 @@ function classifyResponse( return semantics.ID.responseCategoryFromKey(key); } -function dotsFromStimCode(stimCode: string, salience: { high: number; low: number }): Array<{ loc: "A" | "B"; luminance: number }> { +function dotsFromStimCode(stimCode: string, salience: { high: number; low: number }, mode: "visual" | "audiovisual"): Array<{ loc: "A" | "B"; luminance: number }> { const [a, b] = normalizeStimCode(stimCode).split(""); const dots: Array<{ loc: "A" | "B"; luminance: number }> = []; if (a !== "x") dots.push({ loc: "A", luminance: a === "H" ? salience.high : salience.low }); - if (b !== "x") dots.push({ loc: "B", luminance: b === "H" ? salience.high : salience.low }); + if (mode === "visual" && b !== "x") dots.push({ loc: "B", luminance: b === "H" ? salience.high : salience.low }); return dots; }